webXMPP

Check-in [3518f8e97d]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Optimize roster; dumping the whole thing on any single change was a waste of data. Instead, as much as possible send deltas, and let the JS side deal with building the latest selectable destinations (including their ordering).
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | master | trunk
Files: files | file ages | folders
SHA3-256: 3518f8e97d067dfacf0a198116dccc94008998599ee74d90f48639a2d40020e3
User & Date: web 2020-05-31 22:42:16
Context
2020-05-31
22:47
Regression; include nickname in parens w. selected correspondent Leaf check-in: 121320eb4f user: web tags: master, trunk
22:42
Optimize roster; dumping the whole thing on any single change was a waste of data. Instead, as much as possible send deltas, and let the JS side deal with building the latest selectable destinations (including their ordering). check-in: 3518f8e97d user: web tags: master, trunk
18:30
Continuing work on dealing with spurious server disconnects check-in: 4f3cfc408c user: web tags: master, trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to get.py.

12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55

56
57
58
59
60
61
62
...
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
...
143
144
145
146
147
148
149




150

151

152
153








154
155
156









157
158
159
160
161
162
163
164
165
166
167
#	keys, with appropriate text.
#  /config.json
#	Return JSON of configuration with parameters the
#	browser's JS should use in communicating with this
#	server.
import sys, os, threading
import chore

# A little static function to order accounts based on status
# Each argument is a (acct-name, acct-status) tuple
def _order_stat(a, b):
    # Order by name, status
    aname = a[0]
    astat = a[1]
    bname = b[0]
    bstat = b[1]

    # WTF?
    if aname == bname:
	return 0

    # If tie on status, order by name
    if astat == bstat:
	# Offline XMPP after SMS
	if ':' in aname:
	    if ':' not in bname:
		return -1
	elif ':' in bname:
	    return 1

	# Then alphabetical
	return cmp(aname,bname)

    # Available, away, and then presumably
    #  extended away of some sort
    for k in (("available", "chat"), ("away",), ("xa",), ("dnd",)):
	if astat in k:
	    return -1
	if bstat in k:
	    return 1

    # Two statuses of no known relationship;
    #  consider them equal and order by name
    return cmp(aname,bname)


class GET_mixin(object):

    # Paths for a GET
    def __init__(self):
	self.dispatchers.append( ("GET", self.send_favicon) )
	self.dispatchers.append( ("GET", self.send_ajax) )
................................................................................
	# Try and send the media file
	return True,self.send_files(fn)

    # Send what we have, based on generation @gen
    def send_current(self, gen, rgen):
	approot = self.server.approot
	user = approot.users[self.user]
	nicks = approot.nicks
	msgs = []
	for tup in user.msgs:
	    if tup[0] >= gen:
		d = {}
		d["rx"] = tup[1]
		# TBD, more handling of nicknames
		them = d["them"] = tup[2]
................................................................................

	# Here's the messages we have right now
	resp = { "serial": user.serial,
	    "gen": user.gen, "rgen": user.rgen,
	    "msgs": msgs }
	user.serial += 1





	# Add a roster update?

	if rgen != user.rgen:

	    uroster = []
	    for name in user.roster:








		uroster.append(
		 (name, user.status.get(name, "offline"), nicks.get(name)) )
	    uroster.sort(cmp = _order_stat)









	    resp["roster"] = uroster
	    sys.stderr.write("%s: new roster\n" % (user.name,))
	else:
	    resp["roster"] = tuple()

	# Send off our json encoding of the response
	sys.stderr.write("response to %s: %s\n" % (user.name, resp))
	return self.send_json(resp)

    # Request for messages, when generation passes
    #  "gen" argument, optionally with timeout argument.







<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
>







 







<







 







>
>
>
>

>

>
|
<
>
>
>
>
>
>
>
>
|
<
<
>
>
>
>
>
>
>
>
>
|
<
<
<







12
13
14
15
16
17
18





































19
20
21
22
23
24
25
26
..
88
89
90
91
92
93
94

95
96
97
98
99
100
101
...
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121

122
123
124
125
126
127
128
129
130


131
132
133
134
135
136
137
138
139
140



141
142
143
144
145
146
147
#	keys, with appropriate text.
#  /config.json
#	Return JSON of configuration with parameters the
#	browser's JS should use in communicating with this
#	server.
import sys, os, threading
import chore





































import pdb

class GET_mixin(object):

    # Paths for a GET
    def __init__(self):
	self.dispatchers.append( ("GET", self.send_favicon) )
	self.dispatchers.append( ("GET", self.send_ajax) )
................................................................................
	# Try and send the media file
	return True,self.send_files(fn)

    # Send what we have, based on generation @gen
    def send_current(self, gen, rgen):
	approot = self.server.approot
	user = approot.users[self.user]

	msgs = []
	for tup in user.msgs:
	    if tup[0] >= gen:
		d = {}
		d["rx"] = tup[1]
		# TBD, more handling of nicknames
		them = d["them"] = tup[2]
................................................................................

	# Here's the messages we have right now
	resp = { "serial": user.serial,
	    "gen": user.gen, "rgen": user.rgen,
	    "msgs": msgs }
	user.serial += 1

	# Get a snapshot on first use of each generation
	if user.rgen not in user.rosters:
	    user.rosters[user.rgen] = dict(user.status)

	# Add a roster update?
	rupdates = []
	if rgen != user.rgen:
	    oroster = user.rosters.get(rgen) or {}
	    nroster = user.rosters[user.rgen]


	    nicks = approot.nicks
	    for u,st in oroster.iteritems():
		# Something left roster
		if u not in nroster:
		    rupdates.append( ("-", u) )

		# New status
		nst = nroster[u]


		if st != nst:
		    rupdates.append( ("=", u, nst) )
	    for u in nroster:
		if u not in oroster:
		    nick = nicks.get(u)
		    if not nick:
			nick = u
		    rupdates.append( ("+", u, nick, nroster[u]) )

	resp["roster"] = rupdates




	# Send off our json encoding of the response
	sys.stderr.write("response to %s: %s\n" % (user.name, resp))
	return self.send_json(resp)

    # Request for messages, when generation passes
    #  "gen" argument, optionally with timeout argument.

Changes to js/ui.js.

88
89
90
91
92
93
94



95
96
97
98
99
100
101
...
292
293
294
295
296
297
298





























































































































299
300
301
302
303
304
305
...
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
// Matching URL's in texts
const URLpat =
/(\b(https?|ftp):\/\/[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])/gi ;

// Mapping for nicknames
const name_nick = new Map(), nick_name = new Map();




// Returning online (network change, cell->wifi, etc.)
function goOnline() {
    if (curReq != null) {
	curReq.abort();
	curReq = null;
    }

................................................................................
    }
    if (mn < 10) {
	mn = '0' + String(mn);
    }
    return (hr + ":" + mn);
}






























































































































// WebXMPP server has given us messages
function gotMsgs() {

    // Connection progress
    if ((curReq == null) || (curReq.readyState != 4)) {
	return;
    }
................................................................................
	lastTM = tm;
    }

    // Updated roster?  This changes the available selections
    //  in the pulldown.
    rGen = resp.rgen;
    if (resp.roster.length > 0) {
	// Clear old
	ourDests.length = 0;

	// Build new
	// First one says "whoever last sent to us", the rest
	//  enumerate buddies in our rosters.
	ourDests[0] = new Option("--", "--", true, false);
	for (let i = 0; i < resp.roster.length; ++i) {
	    // Each tuple is (acct-name, acct-status)
	    const tup = resp.roster[i];
	    const who = tup[0]
	    const st = tup[1]
	    const nick = tup[2]

	    // Label +name for available, -name for away,
	    //  and .name for extended away.  While not an XMPP
	    //  status, "offline" is a leading blank for a roster
	    //  member who is not online.
	    let nm = null;
	    if ((st == "available") || (st == "chat")) {
		nm = "+ " + who;
	    } else if (st == "away") {
		nm = "- " + who;
	    } else if (st == "xa") {
		nm = ". " + who;
	    } else if (st == "dnd") {
		nm = "* " + who;
	    } else {
		nm = "  " + who;
	    }

	    // Add on nickname, if any
	    if (nick) {
		nm = nm + " (" + nick + ")";
		name_nick.set(who, nick);
		nick_name.set(nick, who);
	    }

	    // Another dest we could pick
	    ourDests[i+1] = new Option(nm, who, false, false);
	}
    }

    // No messages
    if (resp.msgs.length == 0) {
	moreMessages();
	return;
    }







>
>
>







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







|
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<







88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
...
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
...
478
479
480
481
482
483
484
485








































486
487
488
489
490
491
492
// Matching URL's in texts
const URLpat =
/(\b(https?|ftp):\/\/[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])/gi ;

// Mapping for nicknames
const name_nick = new Map(), nick_name = new Map();

// Name to status last seen
const name_stat = new Map();

// Returning online (network change, cell->wifi, etc.)
function goOnline() {
    if (curReq != null) {
	curReq.abort();
	curReq = null;
    }

................................................................................
    }
    if (mn < 10) {
	mn = '0' + String(mn);
    }
    return (hr + ":" + mn);
}

// Tell the preferred order of e1 WRT e2
function sort_stat(e1, e2) {
    // Each is [name, st, nickname]
    const e1nm = e1[0];
    const e1st = e1[1];
    const e2nm = e2[0];
    const e2st = e2[1];

    // Shouldn't happen
    if (e1nm == e2nm) {
	return(0);
    }

    // Tie on status, order by name
    if (e1st == e2st) {
	// Offline XMPP after SMS
	if (e1st.includes(':')) {
	    if (!e2st.includes(':')) {
		return -1;
	    }
	} else if (e2st.includes(':')) {
	    return 1;
	}

    } else {

	// Available, away, then extended away
	for (let k of
		[ ["available", "char"], ["away"],
		  ["xa"], ["dnd"] ]) {
	    if (k.indexOf(e1st) >= 0) {
		return -1;
	    }
	    if (k.indexOf(e2st) >= 0) {
		return 1;
	    }
	}
    }

    // Tied except for name
    return (e1nm < e2nm) ? -1 : 1;
}

// The server's telling us about new roster state
function update_roster(rupdates) {
    // First apply each change to our own copy of the roster
    let nm = null, nick = null, st = null;
    for (let tup of rupdates) {
	const op = tup[0];

	// Add or completely update an entry
	if (op == "+") {
	    nm = tup[1];
	    nick = tup[2];
	    st = tup[3];
	    name_nick.set(nm, nick);
	    nick_name.set(nick, nm);
	    name_stat.set(nm, st);
	    continue;
	}

	// Delete an entry
	if (op == "-") {
	    nm = tup[1];
	    nick = name_nick.get(nm);
	    name_nick.delete(nm);
	    nick_name.delete(nick);
	    name_stat.delete(nm);
	    continue;
	}

	// Update status
	if (op == "=") {
	    nm = tup[1];
	    st = tup[2];
	    name_stat.set(nm, st);
	    continue;
	}
	console.log("Unknown op: " + op);
    }

    // Assemble the latest roster
    let roster = [];
    for (nm of name_stat.keys()) {
	roster.push( [nm, name_stat.get(nm), name_nick.get(nm)] );
    }

    // Try to bring useful ones to the front
    roster.sort(sort_stat);

    // Clear old
    ourDests.length = 0;

    // Build new
    // First one says "whoever last sent to us", the rest
    //  enumerate buddies in our rosters.
    ourDests[0] = new Option("--", "--", true, false);
    for (let i = 0; i < roster.length; ++i) {
	// Each tuple is (acct-name, acct-status)
	const tup = roster[i];
	const who = tup[0]
	const st = tup[1]
	const nick = tup[2]

	// Label +name for available, -name for away,
	//  and .name for extended away.  While not an XMPP
	//  status, "offline" is a leading blank for a roster
	//  member who is not online.
	let nm = null;
	if ((st == "available") || (st == "chat")) {
	    nm = "+ " + who;
	} else if (st == "away") {
	    nm = "- " + who;
	} else if (st == "xa") {
	    nm = ". " + who;
	} else if (st == "dnd") {
	    nm = "* " + who;
	} else {
	    nm = "  " + who;
	}

	// Another dest we could pick
	ourDests[i+1] = new Option(nm, who, false, false);
    }
}
// WebXMPP server has given us messages
function gotMsgs() {

    // Connection progress
    if ((curReq == null) || (curReq.readyState != 4)) {
	return;
    }
................................................................................
	lastTM = tm;
    }

    // Updated roster?  This changes the available selections
    //  in the pulldown.
    rGen = resp.rgen;
    if (resp.roster.length > 0) {
	update_roster(resp.roster);








































    }

    // No messages
    if (resp.msgs.length == 0) {
	moreMessages();
	return;
    }

Changes to user.py.

1
2
3
4
5
6
7
8
9
10
11
12
13
..
25
26
27
28
29
30
31

32
33
34
35
36
37
38
..
43
44
45
46
47
48
49


50
51
52
53
54
55
56
#
# user.py
#	User state
#
import threading, time
from chore.utils import Exclusion
import acct_xmpp, acct_sms

# One of these is spun up when a user first authenticates.  It caches
#  the user/pw, and fires up threads to watch XMPP for each configured
#  server.
#
# approot - Top-level server state
................................................................................
# msgs[] - Current list of tuples representing messages being
#	displayed out at the browser.  There are config["nmsg"]
#	of them.  Each is:
#	 (generation#, from, to, body, mime-type, media-name)
# pending[] - List of clients waiting (Ajax) for new messages
#	Each is a tuple (timeout, semaphore, client-id, www_handler)
# roster{} - Map from recipient to the Account they're under

# status{} - Map from account name to their status
#	(it was under the User directly, but you can get back
#	 presence messages before the user shows up in the roster)
# serial - Running counter, so clients can detect dup (from cache)
#	completions.
# pushkeys[] - Set of active Google messaging endpoints
# host - Record host at which they reach us
................................................................................
	self.name = user
	self.accounts = accounts
	self.activity = None
	self.timeout = None
	self.exclusion = Exclusion()
	self.active = {}
	self.roster = {}


	self.status = {}
	self.pushkeys = set()
	self.serial = 1
	self.host = None

	# Initial generation of content;
	#  roster current, and





|







 







>







 







>
>







1
2
3
4
5
6
7
8
9
10
11
12
13
..
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
..
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
#
# user.py
#	User state
#
import threading, time
from chore.utils import Exclusion, TimedDict
import acct_xmpp, acct_sms

# One of these is spun up when a user first authenticates.  It caches
#  the user/pw, and fires up threads to watch XMPP for each configured
#  server.
#
# approot - Top-level server state
................................................................................
# msgs[] - Current list of tuples representing messages being
#	displayed out at the browser.  There are config["nmsg"]
#	of them.  Each is:
#	 (generation#, from, to, body, mime-type, media-name)
# pending[] - List of clients waiting (Ajax) for new messages
#	Each is a tuple (timeout, semaphore, client-id, www_handler)
# roster{} - Map from recipient to the Account they're under
# rosters{} - Map from roster generation to roster at that generation
# status{} - Map from account name to their status
#	(it was under the User directly, but you can get back
#	 presence messages before the user shows up in the roster)
# serial - Running counter, so clients can detect dup (from cache)
#	completions.
# pushkeys[] - Set of active Google messaging endpoints
# host - Record host at which they reach us
................................................................................
	self.name = user
	self.accounts = accounts
	self.activity = None
	self.timeout = None
	self.exclusion = Exclusion()
	self.active = {}
	self.roster = {}
	# Old versions of roster, up to 45 minutes ago
	self.rosters = TimedDict(45*60)
	self.status = {}
	self.pushkeys = set()
	self.serial = 1
	self.host = None

	# Initial generation of content;
	#  roster current, and