metadata

Check-in [1888e3b612]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Initial check-in
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256:1888e3b612d55e6db9071f648802843c1a24ae4818a540f04b39c5bdab791026
User & Date: vandys 2018-06-18 22:05:12
Context
2018-06-18
22:23
Initial run check-in: d4b4a6924b user: vashon tags: trunk
22:05
Initial check-in check-in: 1888e3b612 user: vandys tags: trunk
22:04
initial empty check-in check-in: 08abbebd6a user: vandys tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Added etc/config.







































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

serve http
    port 2020

# Radio Paradise
stream rp http://stream-tx3.radioparadise.com:80/mp3-32

# Voice of Vashon
stream vov http://VoiceOfVashon.serverroom.us:4256

# KVMR and its web-only "younger" format
# No metadata
# stream kvmr http://live.kvmr.org:8000/dial
# stream kvmrx http://live.kvmr.org:8000/kvmr2-mp3-128

# Seattle Dance C89.5
stream c89.5 http://knhc-ice.streamguys1.com:80/live

# LaGrosse Radio Paris - Rock
stream lagr http://hd.lagrosseradio.info/lagrosseradio-rock-192.mp3

# M2 Radio Paris - Pop
stream m2radio http://neo.m2stream.fr:8000/m2hit-256.mp3

# Antenne Bayern Top-40
stream anten http://mp3channels.webradio.antenne.de:80/top-40

# KEXP
stream kexp http://live-mp3-32.kexp.org/

# KALW
stream kalw http://live.str3am.com:2430/kalw

# WFMU
stream wfmu http://stream0.wfmu.org/freeform-32k

Added get.py.



































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
#
# get.py
#	Mixin to implement HTML GET operations
#
#
import os, sys, urllib, time

# The GET part of our handling
class GET_mixin(object):

    # Configure our WPlayer GET treatment
    def __init__(self):

	# GET handlers
	self.dispatchers.append( ("GET", self.send_metadata) )

    # "/"; Web page with player
    def send_top(self):
	# Nope, not user friendly
	return self.send_error(404)

    # Serve metadata for a stream
    #
    # /metadata/<host-sym>.json
    def send_metadata(self):
	pp = self.paths
	if (not pp) or (len(pp) != 2) or (pp[0] != "metadata") \
		or (not pp[1].endswith(".json")):
	    return False,None
	approot = self.server.approot

	# Get URL
	who = pp[1][:-5]
	strm = approot.streams.get(who)
	if strm is None:
	    # Unknown host
	    return True,self.send_error(421)

	# Which generation?
	if self.vals and ("gen" in self.vals):
	    try:
		gen = int(self.vals["gen"])
	    except:
		return True,self.send_error(400)
	else:
	    gen = 0

	# Handle this request for this stream
	return strm.serve(gen)

Added main.py.









































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
#
# main.py
#	Provide metadata updates for one or more radio stations
#
# We pull the slowest stream we know, and scrape the Icecast
#  metadata from it, then hand that out to any requesting clients
#
import sys, time
from chore.handlers import Chore_Handler
from get import GET_mixin
from stream import Stream
import pdb

# Tie our various handlers together
class App_Handler(Chore_Handler, GET_mixin):
    def __init__(self, conn, tup, approot):
	Chore_Handler.__init__(self, conn, tup, approot,
	    (GET_mixin.__init__,))

# Load our configuration file
#
# This includes configuring our config file elements,
#  then processing the supplied file.
def load_cfg(fn):

    # A configurator
    c = chore.config.Config()

    # Let the web network side add its config entries
    chore.www.add_config(c)

    # Sources of metadata
    #  stream <symbol> <url>
    # i.e.,
    #  stream rp http://stream-tx3.radioparadise.com:80/mp3-32
    c.args.add( ("stream",) )

    # Parse the input
    cfg = c.load_cfg(fn)

    return cfg

# Root of our app server
class App(chore.server.Server):

    def __init__(self, config):
	global DBlove

	# Let Chore handle most things
	chore.server.Server.__init__(self, config, App_Handler);

	# State per stream
	self.streams = {}

	# Set initial dict contents
	for sym,url in config["stream"]"
	    s = Stream(sym, url)
	    self.streams[sym] = s

if __name__ == "__main__":
    if len(sys.argv) != 2:
	sys.stderr.write("Usage is: %s <config-file>\n" %
	    (sys.argv[0],))
	sys.exit(1)

    # CLI if needed
    while True:
	time.sleep(60)

Added stream.py.































































































































































































































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
#
# stream.py
#	Code for serving metadata on a  given stream
#
import sys, threading, json, urlparse

# HTTP headers to get Icecast metadata
Args = """icy-metadata: 1\r
User-Agent: mplayer-metadata\r
"""


# All the state for a given stream
class Stream(object):
    def __init__(self, sym, url):
	# Sym and URL
	self.sym = sym
	r = urlparse.urlparse(url)
	self.host = r.hostname
	self.port = r.port or 80
	self.path = r.path

	# Connect to the stream
	s = self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
	s.connect( (self.host, self.port) )

	# Current generation of content
	self.gen = 0

	# Current JSON result for the self.gen value
	self.json = None

	# Waiters for new self.gen value
	self.queue = []

	# Thread pulling on the stream
	self.thread = None

    # Pull a line from our socket
    # Not remendously efficient, but tolerable for HTTP headers
    #  (he said hopefully)
    def readline(self):
	s = self.sock
	res = ""
	while True:
	    try:
		c = s.recv(1)
	    except:
		return None
	    if not c:
		return None

	    # \r\n, network-style line ending
	    if c == '\r':
		continue
	    if c == '\n':
		return res
	    res += c

    # Wake up all waiters
    def wakeups(self):
	ws = tuple(self.waiters)
	del self.waters[:]
	while ws:
	    sema = self.waiters.pop()
	    sema.release()

    # Pull from a stream with Icecast metadata.
    # Because Icecast metadata is kind of a hairball of legacy and
    #  modern, we just talk TCP to it ourselves.
    # This function only returns on errors like aborted streams.
    # On idle timeout, it simply exits after clearing itself
    #  from the stream state.
    def watch(self, gen):
	gen = gen or 0

	# Request it
	s.send("GET %s?gen=%d HTTP/1.0\r\n%s\r\n" %
	    (self.path, gen, Args))

	# Assemble the HTTP response headers.  We'll likely have
	#  more than that, which will be passed down to the loop
	#  below.
	res = ""
	headers = {}
	hdone = False
	while not hdone:
	    l = self.readline()
	    if l is None:
		return
	    if not l:
		# Empty line, followed by HTTP response body
		break

	    # Malformed line
	    if ':' not in l:
		continue

	    # key: value...
	    idx = l.index(':')
	    k = l[:idx].strip().lower()
	    v = l[idx+1:].strip()

	    # Register another header line
	    res[k] = v

	# See if this stream does indeed supply Icecast metadata
	if "icy-metaint" not in headers:
	    sys.stderr.write("No icy-metaint for %s\n" % (self.host,))
	    return
	try:
	    mi = headers["icy-metaint"]
	    intvl = int(mi)
	except:
	    sys.stderr.write("Corrupt icy-metaint for %s: %s\n" %
		(self.host, mi))
	    return
	if intvl < 512:
	    sys.stderr.write("Spurious icy-metaint for %s: %d\n" %
		(self.host, intvl))
	    return

	# Now scoop up parcels of "intvl", then grab a metadata update
	while True:
	    # Skip this much
	    try:
		buf = s.recv(intvl)
	    except:
		return
	    if len(buf) != intvl:
		sys.stderr.write("Skipping data got %d\n" % (len(buf),))
		return

	    # An initial byte is the number of 16-byte chunks in
	    #  the stream.
	    try:
		c = s.recv(1)
	    except:
		return
	    if not c:
		sys.stderr.write("No metadata size byte\n")
		return
	    paksize = (ord(c) << 4)

	    # No metadata update needed yet
	    if not paksize:
		continue
	    try:
		buf = s.recv(paksize)
	    except:
		return
	    if len(buf) != paksize:
		sys.stderr.write("metadata got %d\n" % (len(buf),))
		return
	    buf = buf.strip()

	    # Decode metadata into a dict for JSON purposes
	    res = {}
	    for s in buf.split(';'):
		if '=' not in s:
		    sys.stderr.write("Malformed %s from %s\n" %
			(s, self.host))
		    continue
		idx = s.index('=')
		k = s[:idx].strip()
		v = s[idx+1:].strip()
		if v[0] == "'":
		    v = v[1:]
		    if v[-1] == "'":
			v = v[:-1]
		res[k] = v
	    
	    if not res:
		sys.stderr.write("%s no usable metadata\n" % (self.host,))
		continue

	    # Convert to a JSON string
	    self.gen += 1
	    self.json = json.dumps(res)
	    self.wakeups()

    # We're running on a thread for a given HTTP request.
    # Send back metadata, now, or in the future when it
    #  changes (long polling)
    def serve(self, req, gen):

	# If we already have the answer they want, send it back
	if self.gen and (gen < self.gen):
	    return True,req.send_result(self.json, "application/json")

	# Put ourselves on the queue now; we'll sleep once we've
	#  perhaps started up a service thread
	s = threading.Semaphore(0)
	self.queue.append(s)

	# Start a thread if needed
	if self.thread is None:
	    self.thread = t = \
		threading.Thread(target=self.watch, args=(gen,))
	    t.start()

	# Wait for the thread to find metadata and wake us up
	s.acquire()

	# We're back, send the contents
	assert self.json
	return True,req.send_result(self.json, "application/json")