wusenet

Check-in [9c312f1ad1]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Start coding up NNTP connection and initial web screens
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | master | trunk
Files: files | file ages | folders
SHA3-256:9c312f1ad1f41672b07bfafb432ef8aeeeb4b4970c171e64f4718c3a44d054d9
User & Date: ajv-899-334-8894@vsta.org 2015-04-07 00:54:31
Context
2015-04-07
00:57
Pre-github check-in: 526aa10093 user: ajv-899-334-8894@vsta.org tags: master, trunk
00:54
Start coding up NNTP connection and initial web screens check-in: 9c312f1ad1 user: ajv-899-334-8894@vsta.org tags: master, trunk
2015-04-06
17:17
Initial checkin, git source control check-in: b81b20ee38 user: ajv-899-334-8894@vsta.org tags: master, trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to get.py.

    13     13   #  /kill
    14     14   #	Clickable delete, radio from/subject, then pattern to kill
    15     15   #  /group.name
    16     16   #	Fetches articles for group.name, NART oldest not yet read
    17     17   #  /group.name/<index>
    18     18   #	Reads article for group.name at NNTP server <index>
    19     19   #	Button for reformat, telescope quoting, follow-up
    20         -#
    21     20   # 
    22         -import os, copy
    23     21   
    24     22   # Number of articles to display at one time
    25     23   NART = 50
    26     24   
    27         -# These are file extensions which should have their bits served
    28         -#  directly, literally.
    29         -Litfiles = (".mp3", ".jpg", ".gif", ".js", ".wav", ".txt")
    30         -
    31         -# These are HTML source files
    32         -HTMLfiles = (".htm", ".html")
    33         -
    34         -# Factor out constructing the HTML5 element to play an MP3
    35         -def mp3player(path, id=None):
    36         -
    37         -    # Optional ID for overall element
    38         -    if id:
    39         -	idstr = ' id="%s"' % (id,)
    40         -    else:
    41         -	idstr = ''
    42         -
    43         -    # Player
    44         -    res = """<span%s>Play: <audio controls>
    45         -<source src="%s" type="audio/mpeg" />
    46         -<span>No player</span>
    47         -</audio></span>""" % (idstr, path)
    48         -
    49         -    return res
    50         -
    51     25   class GET_Mixin(object):
    52     26   
    53         -    # Top level of TIS; current status & available states
           27  +    # "/"; groups list
    54     28       def send_top(self):
    55         -	tis = self.server
    56         -	buf = '<html><title>TIS Management - %s</title><body>\n' % \
    57         -	    (tis.config["org"],)
    58         -	buf += '<form action="." method="post"><table border=1>\n'
    59         -	for idx,state in enumerate(tis.states):
    60         -	    # Each State gets two table rows; the first lets us
    61         -	    #  choose the current state, the second gives us
    62         -	    #  our buttons to edit/clear/etc.
    63         -	    atcur = (idx == tis.curstate)
    64         -
    65         -	    # First row, radio checkbox and title
    66         -	    buf += '<tr>\n'
    67         -	    buf += ' <td><input type="radio" name="stateset"'
    68         -	    buf += ' value=%d' % (idx,)
    69         -	    if atcur:
    70         -		buf += ' checked'
    71         -	    if (not atcur) and (not state.configured()):
    72         -		buf += ' disabled'
    73         -	    buf += '></td>\n'
    74         -	    buf += ' <td><h3>State#%d - %s</h3></td>' % \
    75         -		(idx, state.name or "(empty)")
    76         -	    buf += ' <td><i>alert level %d</i></td>\n' % \
    77         -		(state.get_alert(),)
    78         -	    buf += '</tr>\n'
    79         -
    80         -	    # Second row, edit/clear (first column blank)
    81         -	    buf += '<tr>\n'
    82         -	    buf += ' <td> </td>\n'
    83         -	    buf += ' <td>\n  <a href="state%d/">edit</a>\n' % (idx,)
    84         -	    buf += ' <input type="submit"'
    85         -	    buf += '  name="clear%d" value="clear"' % (idx,)
    86         -	    if atcur:
    87         -		buf += ' disabled'
    88         -	    buf += '>\n'
    89         -	    buf += '</td></tr>\n'
    90         -
    91         -	    # Separators
    92         -	    for x in xrange(2):
    93         -		buf += '<tr class="greenbar"><td colspan=3 ' \
    94         -		    'class="separator"></td></tr>\n'
    95         -
    96         -	buf += '</table>\n'
    97         -
    98         -	# Apply state change
    99         -	buf += '<input type="submit" name="state" ' \
   100         -	    'value="Change State"><br>\n'
   101         -	buf += 'Revert in <input type="number" step="0.01" ' \
   102         -	    'name="sexpires" value="4" size=3> hours\n'
   103         -	buf += 'Lights off in <input type="number" step="0.01" ' \
   104         -	    'name="lexpires" value="4" size=3> hours\n'
   105         -
   106         -	buf += '</form></body></html>\n'
           29  +	server = self.server
           30  +	nntp = server.nntp
           31  +	buf = '<html><head><title>Your groups - %s</title></head><body>\n' % \
           32  +	    (self.user,)
           33  +
           34  +	# Enumerate user's groups.  We'll get updates from NNTP
           35  +	#  as needed.
           36  +	user = server.users[self.user]
           37  +	for gname,cur in user.groups.iteritems():
           38  +	    group = nntp.group(gname)
           39  +	    if group is None:
           40  +		continue
           41  +	    if group.latest < cur:
           42  +		cur = 0
           43  +	    count = group.latest - cur
           44  +	    buf += '<a href="/%s">%s</a> - %d new<br>\n' % (gname, count)
           45  +	# TBD, subscription management
           46  +	# buf += '<hr>'
           47  +
           48  +	buf += '</body></html>\n'
   107     49   	buf = self.send_result(buf)
   108     50   	return buf
   109     51   
   110     52       # Edit a given state
   111     53       def send_state(self, idx):
   112     54   	tis = self.server
   113     55   	state = tis.states[idx]

Changes to handler.py.

    97     97   	    return self.send401("Authentication required\r\n")
    98     98   
    99     99   	# Do we accept it?
   100    100   	auth = base64.decodestring(auth[6:])
   101    101   	tup = auth.split(":")
   102    102   	if len(tup) != 2:
   103    103   	    return self.send401("Malformed authentication response\r\n")
   104         -	tis = self.server
   105         -	priv = tis.authenticate(tup[0], tup[1])
   106         -	if priv is None:
          104  +	server = self.server
          105  +	ok = server.authenticate(tup[0], tup[1])
          106  +	if not ok:
   107    107   	    return self.send401("Incorrect username or password\r\n")
   108         -	self.priv = priv
          108  +
          109  +	# Record username of this connection
          110  +	self.user = tup[0]
   109    111   
          112  +	# Ok
   110    113   	return True
   111    114   
   112    115       # Send header
   113    116       # Also canonicalize to DOS-style line endings (ew)
   114    117       def send_result(self, buf,
   115    118   	    mtype="text/html", cacheable=False, binary=False):
   116    119   

Changes to main.py.

     1      1   #
     2      2   # main.py
     3      3   #	Main driver for WWW-Usenet interface
     4      4   #
     5      5   import sys, threading, time
     6         -import config
            6  +import config, user
     7      7   from www import http
     8      8   from nntp import NNTP
     9      9   
    10     10   # Root of all Wusenet server state
    11     11   class Wusenet(object):
    12     12   
    13         -    def __init__(self, cfgname, datadir):
           13  +    def __init__(self):
    14     14   
    15     15   	# Load in configuration values; our configuration is
    16     16   	#  simply key/val stored in a dict.
    17         -	c = self.config = config.load_cfg(cfgname)
           17  +	c = self.config = config.load_cfg("etc/config")
    18     18   
    19     19   	# Sanity
    20     20   	if not c.get("server"):
    21     21   	    raise Exception, "No NNTP server configured"
    22     22   	if not c.get("user"):
    23     23   	    raise Exception, "No NNTP username configured"
    24     24   	if not c.get("password"):
    25     25   	    raise Exception, "No NNTP password configured"
    26     26   	if not c.get("groups"):
    27     27   	    raise Exception, "No NNTP groups configured"
    28     28   	if not c.get("port"):
    29     29   	    raise Exception, "No HTTP server port configured"
    30     30   
    31         -	# Instantiate NNTP handling
    32         -	d = self.datadir = datadir
           31  +	# Users who have been loaded
           32  +	self.users = {}
    33     33   
    34     34       # Actually start running
    35     35       def run(self):
    36     36   	# Configure NNTP
    37         -	n = self.nntp = NNTP(c["server"],
    38         -	    c["user"], c["password"], d)
           37  +	n = self.nntp = NNTP(c["server"], c["user"], c["password"])
    39     38   	for g in c["groups"]:
    40     39   	    n.subscribe(g)
    41     40   
    42         -	# Dedicate a thread to NNTP
    43         -	t = threading.Thread(target=n.run)
    44         -	t.start()
    45         -
    46     41   	# Configure HTTP, then run it.  Our main thread
    47     42   	#  thus ends up being the listener for HTTP
    48         -	#  clients.
           43  +	#  clients.  Threads will spin off as clients
           44  +	#  arrive.
    49     45   	h = self.http = http.HTTP(self, c["port"])
    50     46   	h.run()
    51     47   
    52     48       # Authenticate a user
    53     49       def authenticate(self, user, pw):
    54         -	try:
    55         -	    f = open("%s/password", "r")
    56         -	    fpw = f.readline()[:-1]
           50  +
           51  +	# Already seen; use our in-memory copy
           52  +	if user in self.users:
           53  +	    return self.users[user].pass == pw
           54  +
           55  +	# Scan our filesystem table
           56  +	f = open("etc/accounts", "r")
           57  +	for l in f:
           58  +	    l = l.strip()
           59  +	    if l.startswith("#"):
           60  +		continue
           61  +	    if not l:
           62  +		continue
           63  +	    tup = l.split()
           64  +	    if len(tup) != 2:
           65  +		continue
           66  +	    if l[0] != user:
           67  +		continue
           68  +
           69  +	    # Verify password against account
           70  +	    if l[1] != pw:
           71  +		break
           72  +
           73  +	    # Good password; load in user
           74  +	    self.users[user] = user.User(user, pw)
    57     75   	    f.close()
    58         -	    return (pw == fpw)
    59         -	except:
    60         -	    return False
           76  +	    return True
           77  +
           78  +	# Unknown user account
           79  +	f.close()
           80  +	return False
    61     81   
    62     82   if __name__ == "__main__":
    63         -    if len(sys.argv) != 3:
    64         -	sys.stderr.write("Usage is: %s <cfg-file> <datadir>\n" %
    65         -	    (sys.argv[0],))
    66         -	sys.exit(1)
    67         -    t = Wusenet(sys.argv[1], sys.argv[2])
           83  +    t = Wusenet()
    68     84       t.run()

Added nntp.py.

            1  +#
            2  +# nntp.py
            3  +#	Server interface into an NNTP server
            4  +#
            5  +# Articles are cached in data/cache/<hash>, where <hash> is the
            6  +#  base64 encoding of a sha1() of the message ID (to avoid
            7  +#  filename issues).  It is left to a cron job to scrub that
            8  +#  directory periodically.
            9  +#
           10  +import time, threading, sha1, nntplib
           11  +
           12  +# Don't ask the NNTP server about the same group at more frequent
           13  +#  intervals than this.
           14  +MINPOLL = 30
           15  +
           16  +# Close our NNTP connection after this many minutes of idleness
           17  +IDLE = 2
           18  +
           19  +# State for a single Usenet group
           20  +#
           21  +# nntp - Link to the NNTP instance using us
           22  +# name - Our Usenet group name
           23  +# when - Time when we last updated from the NNTP server
           24  +# first/last - Article indices range from NNTP server
           25  +class Group(object):
           26  +    def __init__(self, nntp, gname):
           27  +	self.nntp = nntp
           28  +	self.name = gname
           29  +	self.when = self.first = self.last = None
           30  +
           31  +    # Update first/last if needed
           32  +    def poll(self):
           33  +	when = self.when
           34  +	if when is None:
           35  +	    needed = True
           36  +	else:
           37  +	    now = time.time()
           38  +	    needed = (now - when) > MINPOLL*60
           39  +
           40  +	# What we have is still good enough
           41  +	if not needed:
           42  +	    return
           43  +
           44  +	# Serialize
           45  +	nntp = self.nntp
           46  +	nntp.lock()
           47  +
           48  +	# Raced
           49  +	if group.when != when:
           50  +	    nntp.unlock()
           51  +	    return self.poll()
           52  +
           53  +	# Establish NNTP server connection
           54  +	if not nntp.connect():
           55  +	    nntp.unlock()
           56  +	    return
           57  +	conn = nntp.conn
           58  +
           59  +	# Get dope on group
           60  +	try:
           61  +	    resp, count, first, last, name = conn.group(nm)
           62  +	    ok = True
           63  +	except:
           64  +	    ok = False
           65  +
           66  +	# If no network error, update the group
           67  +	if ok:
           68  +	    self.first = first
           69  +	    self.last = last
           70  +	    self.when = now
           71  +
           72  +	nntp.unlock()
           73  +
           74  +# All NNTP activity is wrapped up here
           75  +#
           76  +# server - NNTP server we connect to
           77  +# user/pass - Account on the server
           78  +# conn - nntplib.NNTP instance, while connected
           79  +# when - Time when self.conn last used
           80  +class NNTP(object):
           81  +    def __init__(self, server, user, pass):
           82  +
           83  +	# Our NNTP account on the server
           84  +	self.server = server
           85  +	self.user = user
           86  +	self.pass = pass
           87  +
           88  +	# We'll fire up nntplib NNTP instances as needed, then
           89  +	#  close them back down after IDLE minutes.
           90  +	self.conn = None
           91  +	self.last_used = None
           92  +
           93  +	# Keep track of when we last asked about a given group,
           94  +	#  and what we were told at that time
           95  +	self.groups = {}
           96  +
           97  +	# When somebody wants us to do something, they kick this
           98  +	# When somebody wants to get new data for self.groups{}, they
           99  +	#  grab this and then go talk to nntplib.  Thus, we serialize
          100  +	#  on updates, while permitting web requests to be served
          101  +	#  immediately when the cached data suffices.
          102  +	self.sleeping = threading.Semaphore(1)
          103  +
          104  +    # Return a Group instance for this named Usenet group
          105  +    # Mint one on first reference, and update it if it's more than
          106  +    #  MINPOLL minutes out of date.
          107  +    def group(self, gname):
          108  +	groups = self.groups
          109  +
          110  +	# Get our mint the Group for this @gname
          111  +	if gname in groups:
          112  +	    group = groups[gname]
          113  +	else:
          114  +	    groups[gname] = group = Group(self, gname)
          115  +
          116  +	# Possibly cause it to refresh itself
          117  +	group.poll()
          118  +
          119  +	return group
          120  +
          121  +    # If we're not currently in touch with our NNTP server,
          122  +    #  establish a connection
          123  +    # This also "kicks" the notion of when last used,
          124  +    #  which resets the IDLE countdown.
          125  +    # Returns True if a connection is available, False if the
          126  +    #  NNTP server couldn't be reached.
          127  +    def connect(self):
          128  +	# Currently have a connection
          129  +	if self.conn is not None:
          130  +	    self.when = time.time()
          131  +	    return True
          132  +
          133  +	# Open one
          134  +	try:
          135  +	    conn = nntplib.NNTP(self.server,
          136  +		user=self.user, password=self.pass)
          137  +	except:
          138  +	    return False
          139  +
          140  +	# In touch
          141  +	self.conn = conn
          142  +	self.when = time.time()
          143  +	return True
          144  +

Added user.py.

            1  +#
            2  +# user.py
            3  +#	User state
            4  +#
            5  +# One of these is spun up when a user first authenticates.  It caches
            6  +#  the user/pass, and also loads up the saved state of which groups
            7  +#  are subscribed (and which articles we've seen).
            8  +# Subscription and articles read are pushed back to the filesystem.
            9  +# State is also kept for which message ID's have been seen (so postings
           10  +#  to multiple groups are seen once), but that is only held until
           11  +#  server restart.
           12  +#
           13  +class User(object):
           14  +    def __init__(self, user, pass):
           15  +	self.user = user
           16  +	self.pass = pass
           17  +	self.seen = set()
           18  +	self.load_groups()
           19  +	# TBD: killfile, signature
           20  +
           21  +    # Load record of groups subscribed to, along with
           22  +    #  which article seen in that group
           23  +    def load_groups(self):
           24  +	groups = self.groups = {}
           25  +	fd = None
           26  +	try:
           27  +	    fd = open("data/groups/%s" % (user,), "r")
           28  +	    for l in fd:
           29  +		l = l.strip()
           30  +		# <group>: <last-seen>
           31  +		tup = l.split()
           32  +		if len(tup) != 2:
           33  +		    continue
           34  +		group,seen = tup
           35  +		if not group.endswith(":"):
           36  +		    continue
           37  +		if not seen.isdigit():
           38  +		    continue
           39  +		group = group[:-1]
           40  +		groups[group] = int(seen)
           41  +	except:
           42  +	    pass
           43  +	if fd is not None:
           44  +	    fd.close()
           45  +
           46  +    # Save user state back to filesystem
           47  +    def save_groups(self):
           48  +	fd = open("data/groups/%s.new" % (user,), "w")
           49  +	for group,seen in self.groups.iteritems():
           50  +	    fd.write("%s: %d\n" % (group, seen))
           51  +	fd.close()
           52  +	try:
           53  +	    os.unlink("data/groups/%s.old" % (user,))
           54  +	except:
           55  +	    pass
           56  +	try:
           57  +	    os.rename("data/groups/%s" % (user,),
           58  +		"data/groups/%s.old" % (user,), )
           59  +	except:
           60  +	    pass
           61  +	try:
           62  +	    os.rename("data/groups/%s.new" % (user,),
           63  +		"data/groups/%s" % (user,), )
           64  +	except:
           65  +	    pass