wusenet

Check-in [9c312f1ad1]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Start coding up NNTP connection and initial web screens
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | master | trunk
Files: files | file ages | folders
SHA3-256:9c312f1ad1f41672b07bfafb432ef8aeeeb4b4970c171e64f4718c3a44d054d9
User & Date: ajv-899-334-8894@vsta.org 2015-04-07 00:54:31
Context
2015-04-07
00:57
Pre-github check-in: 526aa10093 user: ajv-899-334-8894@vsta.org tags: master, trunk
00:54
Start coding up NNTP connection and initial web screens check-in: 9c312f1ad1 user: ajv-899-334-8894@vsta.org tags: master, trunk
2015-04-06
17:17
Initial checkin, git source control check-in: b81b20ee38 user: ajv-899-334-8894@vsta.org tags: master, trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to get.py.

13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

54
55
56
57
58
59
60
61
62
63



64
65
66
67
68








69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87



88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
#  /kill
#	Clickable delete, radio from/subject, then pattern to kill
#  /group.name
#	Fetches articles for group.name, NART oldest not yet read
#  /group.name/<index>
#	Reads article for group.name at NNTP server <index>
#	Button for reformat, telescope quoting, follow-up
#
# 
import os, copy

# Number of articles to display at one time
NART = 50

# These are file extensions which should have their bits served
#  directly, literally.
Litfiles = (".mp3", ".jpg", ".gif", ".js", ".wav", ".txt")

# These are HTML source files
HTMLfiles = (".htm", ".html")

# Factor out constructing the HTML5 element to play an MP3
def mp3player(path, id=None):

    # Optional ID for overall element
    if id:
	idstr = ' id="%s"' % (id,)
    else:
	idstr = ''

    # Player
    res = """<span%s>Play: <audio controls>
<source src="%s" type="audio/mpeg" />
<span>No player</span>
</audio></span>""" % (idstr, path)

    return res

class GET_Mixin(object):

    # Top level of TIS; current status & available states

    def send_top(self):
	tis = self.server
	buf = '<html><title>TIS Management - %s</title><body>\n' % \
	    (tis.config["org"],)
	buf += '<form action="." method="post"><table border=1>\n'
	for idx,state in enumerate(tis.states):
	    # Each State gets two table rows; the first lets us
	    #  choose the current state, the second gives us
	    #  our buttons to edit/clear/etc.
	    atcur = (idx == tis.curstate)




	    # First row, radio checkbox and title
	    buf += '<tr>\n'
	    buf += ' <td><input type="radio" name="stateset"'
	    buf += ' value=%d' % (idx,)








	    if atcur:
		buf += ' checked'
	    if (not atcur) and (not state.configured()):
		buf += ' disabled'
	    buf += '></td>\n'
	    buf += ' <td><h3>State#%d - %s</h3></td>' % \
		(idx, state.name or "(empty)")
	    buf += ' <td><i>alert level %d</i></td>\n' % \
		(state.get_alert(),)
	    buf += '</tr>\n'

	    # Second row, edit/clear (first column blank)
	    buf += '<tr>\n'
	    buf += ' <td> </td>\n'
	    buf += ' <td>\n  <a href="state%d/">edit</a>\n' % (idx,)
	    buf += ' <input type="submit"'
	    buf += '  name="clear%d" value="clear"' % (idx,)
	    if atcur:
		buf += ' disabled'



	    buf += '>\n'
	    buf += '</td></tr>\n'

	    # Separators
	    for x in xrange(2):
		buf += '<tr class="greenbar"><td colspan=3 ' \
		    'class="separator"></td></tr>\n'

	buf += '</table>\n'

	# Apply state change
	buf += '<input type="submit" name="state" ' \
	    'value="Change State"><br>\n'
	buf += 'Revert in <input type="number" step="0.01" ' \
	    'name="sexpires" value="4" size=3> hours\n'
	buf += 'Lights off in <input type="number" step="0.01" ' \
	    'name="lexpires" value="4" size=3> hours\n'

	buf += '</form></body></html>\n'
	buf = self.send_result(buf)
	return buf

    # Edit a given state
    def send_state(self, idx):
	tis = self.server
	state = tis.states[idx]







|
<
<




<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<


<
>

|
<
<
<
<
<
<
<
<
>
>
>

<
<
<
<
>
>
>
>
>
>
>
>
|
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
>
>
>
|
<

<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
|







13
14
15
16
17
18
19
20


21
22
23
24
























25
26

27
28
29








30
31
32
33




34
35
36
37
38
39
40
41
42


















43
44
45
46

47















48
49
50
51
52
53
54
55
#  /kill
#	Clickable delete, radio from/subject, then pattern to kill
#  /group.name
#	Fetches articles for group.name, NART oldest not yet read
#  /group.name/<index>
#	Reads article for group.name at NNTP server <index>
#	Button for reformat, telescope quoting, follow-up
# 



# Number of articles to display at one time
NART = 50

























class GET_Mixin(object):


    # "/"; groups list
    def send_top(self):
	server = self.server








	nntp = server.nntp
	buf = '<html><head><title>Your groups - %s</title></head><body>\n' % \
	    (self.user,)





	# Enumerate user's groups.  We'll get updates from NNTP
	#  as needed.
	user = server.users[self.user]
	for gname,cur in user.groups.iteritems():
	    group = nntp.group(gname)
	    if group is None:
		continue
	    if group.latest < cur:
		cur = 0


















	    count = group.latest - cur
	    buf += '<a href="/%s">%s</a> - %d new<br>\n' % (gname, count)
	# TBD, subscription management
	# buf += '<hr>'

















	buf += '</body></html>\n'
	buf = self.send_result(buf)
	return buf

    # Edit a given state
    def send_state(self, idx):
	tis = self.server
	state = tis.states[idx]

Changes to handler.py.

97
98
99
100
101
102
103
104
105
106
107
108
109




110
111
112
113
114
115
116
	    return self.send401("Authentication required\r\n")

	# Do we accept it?
	auth = base64.decodestring(auth[6:])
	tup = auth.split(":")
	if len(tup) != 2:
	    return self.send401("Malformed authentication response\r\n")
	tis = self.server
	priv = tis.authenticate(tup[0], tup[1])
	if priv is None:
	    return self.send401("Incorrect username or password\r\n")
	self.priv = priv





	return True

    # Send header
    # Also canonicalize to DOS-style line endings (ew)
    def send_result(self, buf,
	    mtype="text/html", cacheable=False, binary=False):








|
|
|

<

>
>
>
>







97
98
99
100
101
102
103
104
105
106
107

108
109
110
111
112
113
114
115
116
117
118
119
	    return self.send401("Authentication required\r\n")

	# Do we accept it?
	auth = base64.decodestring(auth[6:])
	tup = auth.split(":")
	if len(tup) != 2:
	    return self.send401("Malformed authentication response\r\n")
	server = self.server
	ok = server.authenticate(tup[0], tup[1])
	if not ok:
	    return self.send401("Incorrect username or password\r\n")


	# Record username of this connection
	self.user = tup[0]

	# Ok
	return True

    # Send header
    # Also canonicalize to DOS-style line endings (ew)
    def send_result(self, buf,
	    mtype="text/html", cacheable=False, binary=False):

Changes to main.py.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

49
50
51
52
53
54

55
56






















57
58
59



60
61
62
63
64
65
66
67
68
#
# main.py
#	Main driver for WWW-Usenet interface
#
import sys, threading, time
import config
from www import http
from nntp import NNTP

# Root of all Wusenet server state
class Wusenet(object):

    def __init__(self, cfgname, datadir):

	# Load in configuration values; our configuration is
	#  simply key/val stored in a dict.
	c = self.config = config.load_cfg(cfgname)

	# Sanity
	if not c.get("server"):
	    raise Exception, "No NNTP server configured"
	if not c.get("user"):
	    raise Exception, "No NNTP username configured"
	if not c.get("password"):
	    raise Exception, "No NNTP password configured"
	if not c.get("groups"):
	    raise Exception, "No NNTP groups configured"
	if not c.get("port"):
	    raise Exception, "No HTTP server port configured"

	# Instantiate NNTP handling
	d = self.datadir = datadir

    # Actually start running
    def run(self):
	# Configure NNTP
	n = self.nntp = NNTP(c["server"],
	    c["user"], c["password"], d)
	for g in c["groups"]:
	    n.subscribe(g)

	# Dedicate a thread to NNTP
	t = threading.Thread(target=n.run)
	t.start()

	# Configure HTTP, then run it.  Our main thread
	#  thus ends up being the listener for HTTP
	#  clients.

	h = self.http = http.HTTP(self, c["port"])
	h.run()

    # Authenticate a user
    def authenticate(self, user, pw):
	try:

	    f = open("%s/password", "r")
	    fpw = f.readline()[:-1]






















	    f.close()
	    return (pw == fpw)
	except:



	    return False

if __name__ == "__main__":
    if len(sys.argv) != 3:
	sys.stderr.write("Usage is: %s <cfg-file> <datadir>\n" %
	    (sys.argv[0],))
	sys.exit(1)
    t = Wusenet(sys.argv[1], sys.argv[2])
    t.run()





|






|



|













|
|




|
<



<
<
<
<


|
>





<
>
|
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

|
<
>
>
>
|


<
<
<
<
|

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

38
39
40




41
42
43
44
45
46
47
48
49

50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76

77
78
79
80
81
82




83
84
#
# main.py
#	Main driver for WWW-Usenet interface
#
import sys, threading, time
import config, user
from www import http
from nntp import NNTP

# Root of all Wusenet server state
class Wusenet(object):

    def __init__(self):

	# Load in configuration values; our configuration is
	#  simply key/val stored in a dict.
	c = self.config = config.load_cfg("etc/config")

	# Sanity
	if not c.get("server"):
	    raise Exception, "No NNTP server configured"
	if not c.get("user"):
	    raise Exception, "No NNTP username configured"
	if not c.get("password"):
	    raise Exception, "No NNTP password configured"
	if not c.get("groups"):
	    raise Exception, "No NNTP groups configured"
	if not c.get("port"):
	    raise Exception, "No HTTP server port configured"

	# Users who have been loaded
	self.users = {}

    # Actually start running
    def run(self):
	# Configure NNTP
	n = self.nntp = NNTP(c["server"], c["user"], c["password"])

	for g in c["groups"]:
	    n.subscribe(g)





	# Configure HTTP, then run it.  Our main thread
	#  thus ends up being the listener for HTTP
	#  clients.  Threads will spin off as clients
	#  arrive.
	h = self.http = http.HTTP(self, c["port"])
	h.run()

    # Authenticate a user
    def authenticate(self, user, pw):


	# Already seen; use our in-memory copy
	if user in self.users:
	    return self.users[user].pass == pw

	# Scan our filesystem table
	f = open("etc/accounts", "r")
	for l in f:
	    l = l.strip()
	    if l.startswith("#"):
		continue
	    if not l:
		continue
	    tup = l.split()
	    if len(tup) != 2:
		continue
	    if l[0] != user:
		continue

	    # Verify password against account
	    if l[1] != pw:
		break

	    # Good password; load in user
	    self.users[user] = user.User(user, pw)
	    f.close()
	    return True


	# Unknown user account
	f.close()
	return False

if __name__ == "__main__":




    t = Wusenet()
    t.run()

Added nntp.py.

































































































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
#
# nntp.py
#	Server interface into an NNTP server
#
# Articles are cached in data/cache/<hash>, where <hash> is the
#  base64 encoding of a sha1() of the message ID (to avoid
#  filename issues).  It is left to a cron job to scrub that
#  directory periodically.
#
import time, threading, sha1, nntplib

# Don't ask the NNTP server about the same group at more frequent
#  intervals than this.
MINPOLL = 30

# Close our NNTP connection after this many minutes of idleness
IDLE = 2

# State for a single Usenet group
#
# nntp - Link to the NNTP instance using us
# name - Our Usenet group name
# when - Time when we last updated from the NNTP server
# first/last - Article indices range from NNTP server
class Group(object):
    def __init__(self, nntp, gname):
	self.nntp = nntp
	self.name = gname
	self.when = self.first = self.last = None

    # Update first/last if needed
    def poll(self):
	when = self.when
	if when is None:
	    needed = True
	else:
	    now = time.time()
	    needed = (now - when) > MINPOLL*60

	# What we have is still good enough
	if not needed:
	    return

	# Serialize
	nntp = self.nntp
	nntp.lock()

	# Raced
	if group.when != when:
	    nntp.unlock()
	    return self.poll()

	# Establish NNTP server connection
	if not nntp.connect():
	    nntp.unlock()
	    return
	conn = nntp.conn

	# Get dope on group
	try:
	    resp, count, first, last, name = conn.group(nm)
	    ok = True
	except:
	    ok = False

	# If no network error, update the group
	if ok:
	    self.first = first
	    self.last = last
	    self.when = now

	nntp.unlock()

# All NNTP activity is wrapped up here
#
# server - NNTP server we connect to
# user/pass - Account on the server
# conn - nntplib.NNTP instance, while connected
# when - Time when self.conn last used
class NNTP(object):
    def __init__(self, server, user, pass):

	# Our NNTP account on the server
	self.server = server
	self.user = user
	self.pass = pass

	# We'll fire up nntplib NNTP instances as needed, then
	#  close them back down after IDLE minutes.
	self.conn = None
	self.last_used = None

	# Keep track of when we last asked about a given group,
	#  and what we were told at that time
	self.groups = {}

	# When somebody wants us to do something, they kick this
	# When somebody wants to get new data for self.groups{}, they
	#  grab this and then go talk to nntplib.  Thus, we serialize
	#  on updates, while permitting web requests to be served
	#  immediately when the cached data suffices.
	self.sleeping = threading.Semaphore(1)

    # Return a Group instance for this named Usenet group
    # Mint one on first reference, and update it if it's more than
    #  MINPOLL minutes out of date.
    def group(self, gname):
	groups = self.groups

	# Get our mint the Group for this @gname
	if gname in groups:
	    group = groups[gname]
	else:
	    groups[gname] = group = Group(self, gname)

	# Possibly cause it to refresh itself
	group.poll()

	return group

    # If we're not currently in touch with our NNTP server,
    #  establish a connection
    # This also "kicks" the notion of when last used,
    #  which resets the IDLE countdown.
    # Returns True if a connection is available, False if the
    #  NNTP server couldn't be reached.
    def connect(self):
	# Currently have a connection
	if self.conn is not None:
	    self.when = time.time()
	    return True

	# Open one
	try:
	    conn = nntplib.NNTP(self.server,
		user=self.user, password=self.pass)
	except:
	    return False

	# In touch
	self.conn = conn
	self.when = time.time()
	return True

Added user.py.



































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
#
# user.py
#	User state
#
# One of these is spun up when a user first authenticates.  It caches
#  the user/pass, and also loads up the saved state of which groups
#  are subscribed (and which articles we've seen).
# Subscription and articles read are pushed back to the filesystem.
# State is also kept for which message ID's have been seen (so postings
#  to multiple groups are seen once), but that is only held until
#  server restart.
#
class User(object):
    def __init__(self, user, pass):
	self.user = user
	self.pass = pass
	self.seen = set()
	self.load_groups()
	# TBD: killfile, signature

    # Load record of groups subscribed to, along with
    #  which article seen in that group
    def load_groups(self):
	groups = self.groups = {}
	fd = None
	try:
	    fd = open("data/groups/%s" % (user,), "r")
	    for l in fd:
		l = l.strip()
		# <group>: <last-seen>
		tup = l.split()
		if len(tup) != 2:
		    continue
		group,seen = tup
		if not group.endswith(":"):
		    continue
		if not seen.isdigit():
		    continue
		group = group[:-1]
		groups[group] = int(seen)
	except:
	    pass
	if fd is not None:
	    fd.close()

    # Save user state back to filesystem
    def save_groups(self):
	fd = open("data/groups/%s.new" % (user,), "w")
	for group,seen in self.groups.iteritems():
	    fd.write("%s: %d\n" % (group, seen))
	fd.close()
	try:
	    os.unlink("data/groups/%s.old" % (user,))
	except:
	    pass
	try:
	    os.rename("data/groups/%s" % (user,),
		"data/groups/%s.old" % (user,), )
	except:
	    pass
	try:
	    os.rename("data/groups/%s.new" % (user,),
		"data/groups/%s" % (user,), )
	except:
	    pass