--- /dev/null
+#Architecture
+
+background.py executed periodically (every 25 min).
+service.py is a constantly-running service, uses postgresql asynchronous notifications to detect when there is work to do.
+See file headers of the above to learn about their responsibilities.
+background.py and service.py act on a "service"; this is sort of a multi-tenant capability. The default service name is "autojoin"; to use a different service name set the environment variable LIVEAUTOJOINSERVICE
+service name is also used to find the database via "~/.pg\_service.conf"
+
+web.py is the Flask (wsgi) web application which handles /authorize, /invite, /ticket.
+
+flow:
+1. user clicks join link (/authorize) in thread sidebar
+2. app validates `thread` argument
+3. app redirects to reddit OAuth2 /api/v1/authorize, state is <service>:<event>
+4. user clicks 'Allow'
+5. reddit redirects to /invite
+6. app creates ticket linked to authorization code and `NOTIFY live_autojoin`
+7. app redirects to /ticket (templates/status.html - auto-refreshing)
+7. service.py retrieves authorization code to create authorization
+8. service.py fills in ticket username using authorization
+9. service.py invites user using admin authorization
+10. service.py accepts invite using authorization
+11. auto-refreshing /ticket starts redirecting back to the live event
+
+# Roadmap/wishlist
+- allowed events: load more than 1 page of search results
+- allowed events: search linkpost self text and "thread directory" wiki page for links, not just linkpost links
+- when accepting admin invite, update reddit\_liveupdate\_event\_contributor table like when inviting users
+- don't mention users in announcements
+- respect sr bans
+- check whether we will succeed (have permission, have scope, not too many outstanding invites) before adding ticket
+- configurable (wiki page) allow/deny list for events
+- invite sr moderators with extra permissions
+- handle LIVEUPDATE\_TOO\_MANY\_INVITES (or whatever)
+- actually report status/errors on /ticket
+- handle no admin permission in `invite\_contributor`
+- ticket processing rate limit (max 3 tickets in 60 seconds) - and if waiting on ratelimit, say so in status.html
+- read modmail (and PMs?) and create tickets for messages with "inviteme" commands
+- sync /live/mine (or allowed threads by\_id) to reddit\_liveupdate\_event table (background.py)
+- include event title in error page's /message/compose link template
+- after accepting admin invite, send PM to event creator (we don't know who created a thread, but we could find out who posted it in /r/livecounting and check if they have `settings` permission) with instructions for adding join link
+- remove everyone's "close" permission?
+- should be open-sourced, but needs deployment documentation and database schema script and pg\_reddit open-sourced first
+
+- find and fix DoS opportunities (something invalid submitted through web.py crashes service.py): better now.
+- send "/u/username has joined this thread" updates: done.
+- sync event contributor _invites_, not just contributors: done
+- decide how to handle LIVEUPDATE\_ALREADY\_CONTRIBUTOR when `invite\_contributor`ing: done
+- speculatively update invite table after `invite\_contributor`, speculatively update contributor table after `accept\_contributor\_invite`: done
--- /dev/null
+# announce.py - announce new users
+
+import select
+import urllib.parse
+import urllib.request
+import urllib.error
+import base64
+import json
+
+from common import connect_db, service_name
+
+POLL_INTERVAL=3600
+
+(cn, cr) = connect_db()
+
+cr.execute("LISTEN live_autojoin")
+
+while True:
+ cn.poll()
+ cn.notifies.clear()
+ work = False
+ while True:
+ cr.execute("BEGIN")
+ # - invited less than 1 minute ago
+ # - no announcement of same user in same event less than 30 days ago
+ # - user has update permission (not banned)
+ # - admin has update permission
+ # - admin authorization has submit scope
+ cr.execute("""SELECT access_token, admin_username, ticket.event_flake, username FROM live_autojoin_ticket AS ticket JOIN live_autojoin_service ON service_name=name JOIN live_autojoin_admin_authorization USING (service_name) JOIN reddit_app_authorization ON authorization_id=id JOIN reddit_liveupdate_event_contributor AS userrel ON userrel."type"='contributor' AND ticket.event_flake=userrel.event_flake AND username=userrel.name JOIN reddit_liveupdate_event_contributor AS adminrel ON adminrel."type"='contributor' AND ticket.event_flake = adminrel.event_flake AND admin_username = adminrel.name WHERE service_name = %s AND status = 'ok' AND updated_at + '1m' > CURRENT_TIMESTAMP AND NOT EXISTS (SELECT * FROM live_autojoin_announcement WHERE for_username = username AND event_flake = ticket.event_flake AND at + '30d' > updated_at) AND has_permission('update', userrel.permissions) AND has_permission('update', adminrel.permissions) AND array_position(scope, 'submit') IS NOT NULL LIMIT 1""", (service_name,))
+ try:
+ [(access_token, admin_username, event_flake, username)] = cr.fetchall()
+ except ValueError:
+ break
+ else:
+ work = True
+ escaped_username = username.replace('_', '\\_')
+ usertext = f'*[\\/u\\/{ escaped_username }](/user/{ escaped_username }) has joined this thread*'
+ body = urllib.parse.urlencode({ 'api_type': 'json', 'body': usertext }).encode('utf-8')
+ req = urllib.request.Request(f'https://oauth.reddit.com/api/live/{event_flake}/update', data=body, method='POST')
+ req.add_header('Authorization', 'Bearer {}'.format(access_token))
+ req.add_header('User-Agent', 'autojoin/0.1.0')
+ try:
+ res = json.load(urllib.request.urlopen(req))
+ except urllib.error.HTTPError as e:
+ if e.code == 403:
+ admin_username = None
+ usertext = None
+ else:
+ raise
+ cr.execute("INSERT INTO live_autojoin_announcement (at, event_flake, for_username, author, body) VALUES (CURRENT_TIMESTAMP, %s, %s, %s, %s)", (event_flake, username, admin_username, usertext))
+ finally:
+ cn.commit()
+
+ cn.poll()
+ if work or len(cn.notifies) > 0:
+ continue
+
+ select.select([cn], [], [], POLL_INTERVAL)
--- /dev/null
+# background.py - jobs that are not time-critical.
+# - refresh access tokens
+# - sync PMs
+# - sync allowed threads (subreddit links)
+# - accept admin invites
+# - sync contributor lists
+# - TODO sync subreddit bans and moderators
+# - TODO sync modmail (to receive joining requests to modmail)
+
+# roadmap
+# - for PM, allowed threads, and modmail sync, create a lightweight version to run frequently to handle new events quickly--distinct from the more expensive "full sync" that is implemented here.
+
+import urllib.request
+import urllib.parse
+
+import json
+import base64
+import re
+
+from common import connect_db, service_name
+
+#
+# refresh tokens
+#
+
+(cn, cr) = connect_db()
+
+def do_refresh_token(client_id, client_secret, refresh_token):
+ body = urllib.parse.urlencode({ 'grant_type': 'refresh_token', 'refresh_token': refresh_token }).encode('utf-8')
+ req = urllib.request.Request('https://www.reddit.com/api/v1/access_token', data=body, method='POST')
+ auth = base64.b64encode('{}:{}'.format(client_id, client_secret).encode('utf-8')).decode('utf-8')
+ req.add_header('Authorization', 'Basic {}'.format(auth))
+ req.add_header('User-Agent', 'autojoin/0.1.0')
+ res = urllib.request.urlopen(req)
+ return json.load(res)
+
+while True:
+ cr.execute("BEGIN")
+ cr.execute("SELECT authorization_id, client_id, client_secret, refresh_token FROM live_autojoin_admin_authorization JOIN reddit_app_authorization ON authorization_id=id JOIN reddit_app USING (client_id) WHERE service_name = %s AND refresh_token IS NOT NULL AND expires < CURRENT_TIMESTAMP + '30min' LIMIT 1 FOR NO KEY UPDATE OF reddit_app_authorization", [service_name])
+ try:
+ [(authorization_id, client_id, client_secret, refresh_token)] = cr.fetchall()
+ except ValueError:
+ cn.rollback()
+ break
+ else:
+ print('refreshing token for authorization {}'.format(authorization_id))
+ new_token = do_refresh_token(client_id, client_secret, refresh_token)
+ cr.execute("UPDATE reddit_app_authorization SET access_token = %s, refresh_token = %s, scope = regexp_split_to_array(%s, ' ')::reddit_app_scope[], expires = CURRENT_TIMESTAMP + make_interval(secs => %s) WHERE id = %s", (new_token['access_token'], new_token['refresh_token'], new_token['scope'], new_token['expires_in'], authorization_id))
+ cn.commit()
+
+#
+# load PMs
+#
+
+def flatten_privatemessage_listing(json_):
+ assert json_['kind'] == 'Listing'
+ for p in json_['data']['children']:
+ assert p['kind'] == 't4'
+ yield p['data']
+ replies = p['data']['replies']
+ if replies:
+ assert replies['kind'] == 'Listing'
+ for r in replies['data']['children']:
+ assert p['kind'] == 't4'
+ yield p['data']
+
+def privatemessage_to_tuple(data):
+ id_ = data['id']
+ parent_id = None
+ if data['parent_id'] is not None:
+ parent_id = data['parent_id'].split('_',maxsplit=1)[1]
+ assert int(parent_id, 36) == data['first_message']
+ created = data['created_utc']
+ sr = data['subreddit']
+ author = None if data['author'] == '[deleted]' else data['author']
+ if data['dest'].startswith('#'):
+ # modmail (destination is subreddit)
+ dest = None
+ dest_is_sr = True
+ else:
+ # destination is user
+ dest = data['dest']
+ dest_is_sr = False
+ subject = data['subject']
+ body = data['body']
+ return (id_, parent_id, created, sr, author, dest, dest_is_sr, subject, body)
+
+cr.execute("BEGIN")
+cr.execute("SELECT sr, access_token FROM live_autojoin_service JOIN live_autojoin_admin_authorization ON name = service_name JOIN reddit_app_authorization ON authorization_id=id JOIN reddit_app ON reddit_app.client_id=reddit_app_authorization.client_id WHERE service_name = %s", [service_name])
+[(sr, access_token)] = cr.fetchall()
+cr.execute('CREATE TEMPORARY TABLE privatemessage_load (LIKE reddit_privatemessage INCLUDING INDEXES)')
+after = None
+while True:
+ if after:
+ rsc = 'https://oauth.reddit.com/message/messages?raw_json=1&limit=100&after={}'.format(after)
+ else:
+ rsc = 'https://oauth.reddit.com/message/messages?raw_json=1&limit=100'
+ req = urllib.request.Request(rsc, method='GET')
+ req.add_header('Authorization', 'Bearer {}'.format(access_token))
+ req.add_header('User-Agent', 'autojoin/0.1.0')
+ res = json.load(urllib.request.urlopen(req))
+ tuples = (privatemessage_to_tuple(d) for d in flatten_privatemessage_listing(res))
+ cr.executemany('INSERT INTO privatemessage_load (id, parent_id, created, sr, author, dest, dest_is_sr, subject, body) VALUES (%s, %s, to_timestamp(%s), %s, %s, %s, %s, %s, %s)', tuples)
+ if 'after' in res:
+ after = res['after']
+ else:
+ break
+cr.execute("INSERT INTO reddit_privatemessage (id, parent_id, created, sr, author, dest, dest_is_sr, subject, body) SELECT id, parent_id, created, sr, author, dest, dest_is_sr, subject, body FROM privatemessage_load ON CONFLICT ON CONSTRAINT reddit_privatemessage_pkey DO NOTHING")
+cr.execute("DROP TABLE privatemessage_load")
+cn.commit()
+
+#
+# build allowed thread list
+# TODO look beyond first page
+# TODO wiki config page for additional threads
+#
+
+def flake_from_url(url_):
+ result = re.fullmatch('https?://[a-z]+\.reddit\.com/live/([a-z0-9]+)/?(?:\?.*)?', url_)
+ return result and result.group(1)
+
+def allowed_threads():
+ req = urllib.request.Request('https://oauth.reddit.com/r/livecounting/search?q=url%3Alive+site%3Areddit.com+self%3Ano&restrict_sr=on&include_over_18=on&sort=new&t=all&limit=100', method='GET')
+ req.add_header('Authorization', 'Bearer {}'.format(access_token))
+ req.add_header('User-Agent', 'autojoin/0.1.0')
+ res = json.load(urllib.request.urlopen(req))
+ flakes = (flake_from_url(thing['data']['url']) for thing in res['data']['children'] if thing['data']['is_self'] is False)
+ return set((f for f in flakes if f))
+
+cr.execute("BEGIN")
+#cr.execute("DELETE FROM live_autojoin_allowed_event WHERE service_name = %s", (service_name,))
+cr.executemany("INSERT INTO live_autojoin_allowed_event (service_name, event_flake) VALUES (%s, %s) ON CONFLICT (service_name, event_flake) DO NOTHING", ((service_name, f) for f in allowed_threads()))
+cn.commit()
+
+# accept admin invites
+
+cr.execute("BEGIN")
+cr.execute("SELECT id, body FROM reddit_privatemessage JOIN live_autojoin_service ON dest = admin_username WHERE author = 'reddit' AND NOT dest_is_sr AND parent_id IS NULL AND subject LIKE 'invitation to contribute to %%' AND NOT EXISTS (SELECT * FROM live_autojoin_admin_invite WHERE privatemessage_id = id AND name = %s)", (service_name,))
+for (id_, body) in cr.fetchall():
+ match = re.search('/live/([a-z0-9]+)/contributors', body)
+ event_flake = match and match.group(1)
+ result = None
+ if event_flake:
+ body = urllib.parse.urlencode({ 'api_type': 'json' }).encode('utf-8')
+ req = urllib.request.Request('https://oauth.reddit.com/api/live/{}/accept_contributor_invite'.format(event_flake), method='POST', data=body)
+ req.add_header('Authorization', 'Bearer {}'.format(access_token))
+ req.add_header('User-Agent', 'autojoin/0.1.0')
+ urllib.request.urlopen(req)
+ result = 'ok'
+ cr.execute("INSERT INTO live_autojoin_admin_invite (privatemessage_id, event_flake, result) VALUES (%s, %s, %s)", [id_, event_flake, result])
+cn.commit()
+
+# load contributor lists
+
+cr.execute("BEGIN")
+cr.execute("SELECT event_flake FROM live_autojoin_allowed_event WHERE service_name = %s", (service_name,))
+cn.commit()
+for (event_flake,) in cr.fetchall():
+ req = urllib.request.Request('https://oauth.reddit.com/live/{}/contributors'.format(event_flake), method='GET')
+ req.add_header('Authorization', 'Bearer {}'.format(access_token))
+ req.add_header('User-Agent', 'autojoin/0.1.0')
+ contributors_lists = json.load(urllib.request.urlopen(req))
+ # endpoint returns two listings, contributors and contributor _invites_, in JSON sequence.
+ # if not a contributor, cannot see contributor invites.
+ # in that case, no JSON sequence--simply the single listing is returned--annoying.
+ if isinstance(contributors_lists, dict):
+ empty_listing = {'kind': 'UserList', 'data': {'children': []}}
+ contributors_lists = [contributors_lists, empty_listing]
+ assert all((l['kind'] == 'UserList' for l in contributors_lists))
+ contributors = ((event_flake, c['name'], '{{{}}}'.format(','.join(c['permissions'])), ty) for (ty, l) in zip(('contributor', 'invite'), contributors_lists) for c in l['data']['children'])
+ cr.execute("BEGIN")
+ cr.execute("DELETE FROM reddit_liveupdate_event_contributor WHERE event_flake = %s", (event_flake,))
+ cr.executemany("INSERT INTO reddit_liveupdate_event_contributor (event_flake, name, permissions, \"type\") VALUES (%s, %s, %s::text[], %s)", contributors)
+ cn.commit()
+ print(event_flake)
+
+# TODO load subreddit bans (and moderators)
+# TODO load modmail for joining requests
+
+cn.close()
--- /dev/null
+import psycopg2
+import os
+
+service_name = os.environ.get('LIVEAUTOJOINSERVICE', 'autojoin')
+
+def connect_db():
+ cn = psycopg2.connect("service={}".format(service_name))
+ cr = cn.cursor()
+ return (cn, cr)
--- /dev/null
+#!/bin/bash
+
+echo "Ok, but why aren't you using a package manager"
+
+tar cz {cert,privkey}.pem {background,service,announce,common,web}.py templates systemd | ssh root@hatnd.acorn.pw tar xzC /opt/live-autojoin
--- /dev/null
+# service.py - invite processing
+
+#- exchange authentication codes
+#- fetch authorization identity to populate ticket username
+#- invite users
+#- accept invites
+
+import select
+import urllib.parse
+import urllib.request
+import urllib.error
+import base64
+import json
+
+from common import connect_db, service_name
+
+POLL_INTERVAL=3600
+
+(cn, cr) = connect_db()
+
+cr.execute("LISTEN live_autojoin")
+
+while True:
+ cn.poll()
+ cn.notifies.clear()
+ work = False
+ while True:
+ cr.execute("BEGIN")
+ cr.execute("SELECT client_id, client_secret, redirect_uri, id, code FROM live_autojoin_service JOIN reddit_app USING (client_id) JOIN live_autojoin_ticket ON name = service_name JOIN reddit_app_authorization_code ON id = authorization_code_id WHERE authorization_id IS NULL AND service_name = %s LIMIT 1 FOR UPDATE OF live_autojoin_ticket, reddit_app_authorization_code", (service_name,))
+ try:
+ [(client_id, client_secret, redirect_uri, authorization_code_id, code)] = cr.fetchall()
+ except ValueError:
+ break
+ else:
+ work = True
+ body = urllib.parse.urlencode({ 'grant_type': 'authorization_code', 'code': code, 'redirect_uri': redirect_uri }).encode('utf-8')
+ req = urllib.request.Request('https://www.reddit.com/api/v1/access_token', data=body, method='POST')
+ auth = base64.b64encode('{}:{}'.format(client_id, client_secret).encode('utf-8')).decode('utf-8')
+ req.add_header('Authorization', 'Basic {}'.format(auth))
+ req.add_header('User-Agent', 'autojoin/0.1.0')
+ res = json.load(urllib.request.urlopen(req))
+ if 'error' in res:
+ if res['error'] == 'invalid_grant':
+ cr.execute("DELETE FROM reddit_app_authorization_code WHERE id = %s", (authorization_code_id,))
+ else:
+ raise ValueError("unrecognized error '{}'".format(res['error']))
+ else:
+ assert res['token_type'] == 'bearer'
+ cr.execute("WITH q1 AS (INSERT INTO reddit_app_authorization (client_id, access_token, scope, expires) VALUES (%s, %s, regexp_split_to_array(%s, ' ')::reddit_app_scope[], CURRENT_TIMESTAMP + make_interval(secs => %s)) RETURNING id) UPDATE reddit_app_authorization_code SET authorization_id = q1.id FROM q1 WHERE reddit_app_authorization_code.id = %s", (client_id, res['access_token'], res['scope'], res['expires_in'], authorization_code_id))
+ finally:
+ cn.commit()
+
+ while True:
+ cr.execute("BEGIN")
+ cr.execute("SELECT reddit_app_authorization.id, access_token, ticket FROM live_autojoin_ticket JOIN reddit_app_authorization_code ON reddit_app_authorization_code.id=authorization_code_id JOIN reddit_app_authorization ON reddit_app_authorization.id = authorization_id WHERE service_name = %s AND username IS NULL AND array_position(scope, 'identity') IS NOT NULL LIMIT 1 FOR UPDATE OF live_autojoin_ticket", (service_name,))
+ try:
+ [(authorization_id, access_token, ticket)] = cr.fetchall()
+ except ValueError:
+ break
+ else:
+ work = True
+ req = urllib.request.Request('https://oauth.reddit.com/api/v1/me', method='GET')
+ req.add_header('Authorization', 'Bearer {}'.format(access_token))
+ req.add_header('User-Agent', 'autojoin/0.1.0')
+ try:
+ res = json.load(urllib.request.urlopen(req))
+ except urllib.error.HTTPError as e:
+ if e.code == 401:
+ cr.execute("DELETE FROM reddit_app_authorization WHERE id = %s", (authorization_id,))
+ else:
+ raise
+ else:
+ cr.execute("UPDATE live_autojoin_ticket SET username = %s WHERE service_name = %s AND ticket = %s", (res['name'], service_name, ticket))
+ finally:
+ cn.commit()
+
+ while True:
+ cr.execute("BEGIN")
+ cr.execute("SELECT access_token, ticket, event_flake, username FROM live_autojoin_ticket LEFT OUTER JOIN LATERAL (SELECT service_name, access_token FROM live_autojoin_admin_authorization JOIN reddit_app_authorization ON id=authorization_id WHERE service_name = live_autojoin_ticket.service_name LIMIT 1) q1 USING (service_name) WHERE service_name = %s AND username IS NOT NULL AND status IS NULL LIMIT 1 FOR NO KEY UPDATE OF live_autojoin_ticket", (service_name,))
+ try:
+ [(access_token, ticket, event, username)] = cr.fetchall()
+ except ValueError:
+ break
+ else:
+ work = True
+ if access_token is None:
+ result = 'noadmin'
+ else:
+ body = urllib.parse.urlencode({ 'api_type': 'json', 'name': username, 'permissions': '+update', 'type': 'liveupdate_contributor_invite' }).encode('utf-8')
+ req = urllib.request.Request('https://oauth.reddit.com/api/live/{}/invite_contributor'.format(event), data=body, method='POST')
+ req.add_header('Authorization', 'Bearer {}'.format(access_token))
+ req.add_header('User-Agent', 'autojoin/0.1.0')
+ try:
+ res = json.load(urllib.request.urlopen(req))
+ except urllib.error.HTTPError as e:
+ if e.code == 403:
+ result = 'noadmin'
+ else:
+ raise
+ else:
+ errors = [er[0] for er in res['json']['errors']]
+ if not errors:
+ result = 'invited'
+ elif errors == ['LIVEUPDATE_ALREADY_CONTRIBUTOR']:
+ result = 'already_contributor_or_invited'
+ else:
+ raise RuntimeError(str(errors))
+ if result == 'invited':
+ cr.execute("DELETE FROM reddit_liveupdate_event_contributor WHERE event_flake = %s AND name = %s", (event, username))
+ cr.execute("INSERT INTO reddit_liveupdate_event_contributor (event_flake, name, permissions, type) VALUES (%s, %s, liveupdate_permission_set '+update', 'invite') ON CONFLICT DO NOTHING", (event, username))
+ cr.execute("UPDATE live_autojoin_ticket SET status = %s, updated_at = CURRENT_TIMESTAMP WHERE service_name = %s AND ticket = %s", (result, service_name, ticket))
+ finally:
+ cn.commit()
+
+ while True:
+ cr.execute("BEGIN")
+ cr.execute("SELECT reddit_app_authorization.id, access_token, ticket, event_flake, username, status FROM live_autojoin_ticket JOIN reddit_app_authorization_code ON reddit_app_authorization_code.id=authorization_code_id JOIN reddit_app_authorization ON reddit_app_authorization.id = authorization_id WHERE service_name = %s AND status IN ('invited', 'already_contributor_or_invited') AND array_position(scope, 'livemanage') IS NOT NULL LIMIT 1 FOR NO KEY UPDATE OF live_autojoin_ticket", (service_name,))
+ try:
+ [(authorization_id, access_token, ticket, event_flake, username, status)] = cr.fetchall()
+ except ValueError:
+ break
+ else:
+ work = True
+ body = urllib.parse.urlencode({ 'api_type': 'json' }).encode('utf-8')
+ req = urllib.request.Request('https://oauth.reddit.com/api/live/{}/accept_contributor_invite'.format(event_flake), data=body, method='POST')
+ req.add_header('Authorization', 'Bearer {}'.format(access_token))
+ req.add_header('User-Agent', 'autojoin/0.1.0')
+ try:
+ res = json.load(urllib.request.urlopen(req))
+ except urllib.error.HTTPError as e:
+ if e.code == 401:
+ cr.execute("DELETE FROM reddit_app_authorization WHERE id = %s", (authorization_id,))
+ else:
+ raise
+ else:
+ errors = [er[0] for er in res['json']['errors']]
+ if not errors:
+ result = 'ok'
+ elif errors == ['LIVEUPDATE_NO_INVITE_FOUND']:
+ result = 'already_contributor' if status == 'already_contributor_or_invited' else None
+ else:
+ raise RuntimeError(str(errors))
+ if result == 'ok':
+ cr.execute("UPDATE reddit_liveupdate_event_contributor SET type = 'contributor' WHERE event_flake = %s AND name = %s", (event, username))
+ cr.execute("NOTIFY live_autojoin") # for announcements
+ cr.execute("UPDATE live_autojoin_ticket SET status = %s, updated_at = CURRENT_TIMESTAMP WHERE service_name = %s AND ticket = %s", (result, service_name, ticket))
+ finally:
+ cn.commit()
+
+ cn.poll()
+ if work or len(cn.notifies) > 0:
+ continue
+
+ select.select([cn], [], [], POLL_INTERVAL)
--- /dev/null
+[Service]
+WorkingDirectory=/opt/live-autojoin
+User=counting
+Group=counting
+Environment=LIVEAUTOJOINSERVICE=autojoin
+Type=simple
+ExecStart=python3 announce.py
+Restart=always
--- /dev/null
+[Service]
+WorkingDirectory=/opt/live-autojoin
+User=counting
+Group=counting
+Environment=LIVEAUTOJOINSERVICE=autojoin
+Type=oneshot
+ExecStart=python3 background.py
--- /dev/null
+[Timer]
+OnActiveSec=0
+OnUnitActiveSec=25min
--- /dev/null
+[Service]
+WorkingDirectory=/opt/live-autojoin
+User=counting
+Group=counting
+Environment=LIVEAUTOJOINSERVICE=autojoin
+Type=simple
+ExecStart=python3 service.py
+Restart=always
--- /dev/null
+[Service]
+WorkingDirectory=/opt/live-autojoin
+User=counting
+Group=counting
+Environment=SCRIPT_NAME=/autojoin LIVEAUTOJOINSERVICE=autojoin
+Type=simple # `notify` if gunicorn >= 20
+ExecStart=/usr/bin/gunicorn3 web:app
--- /dev/null
+[Socket]
+ListenStream=/tmp/live-autojoin.socket
+SocketUser=www-data
+SocketGroup=www-data
--- /dev/null
+[Unit]
+Requires=live-autojoin-cron.timer live-autojoin-service.service live-autojoin-web.socket live-autojoin-announce.service
+
+[Install]
+WantedBy=multi-user.target
--- /dev/null
+<!DOCTYPE html>
+<html>
+ <head>
+ <title>Autojoin: Error</title>
+ </head>
+ <body>
+ <h1>Error: {% block short %}unknown{% endblock %}</h1>
+ {% block long %}<p>Please <a href="{{ privatemessage_url }}">message the moderators</a> for assistance.<p>{% endblock %}
+ </body>
+</html>
--- /dev/null
+<!DOCTYPE html>
+<html>
+ <head>
+ <title>Autojoin: Pending</title>
+ <meta http-equiv="refresh" content="5">
+ </head>
+ <body>
+ <h1>Inviting you to this thread</h1>
+ <p>This should only take a few seconds. If many users are trying to join right now, you might have to wait longer.</p>
+ <p>You will be redirected back to the live thread once you have been added.</p>
+ <p>If this message persists, please <a href="{{ privatemessage_url }}">message the moderators</a> for assistance.</p>
+ </body>
+</html>
--- /dev/null
+from flask import Flask, request, redirect, render_template, url_for
+
+import secrets
+import re
+import urllib.parse
+
+from common import connect_db
+
+DEFAULT_SERVICE = 'autojoin'
+
+app = Flask(__name__)
+
+def make_oauth_url(service_name, client_id, event, redirect_uri):
+ state = '{}:{}'.format(service_name, event)
+ scope = ' '.join(['identity','livemanage'])
+ qs = urllib.parse.urlencode({
+ 'client_id': client_id,
+ 'response_type': 'code',
+ 'state': state,
+ 'redirect_uri': redirect_uri,
+ 'scope': scope,
+ 'duration': 'temporary'
+ })
+ return 'https://www.reddit.com/api/v1/authorize?{}'.format(qs)
+
+def make_privatemessage_url(service_name, event):
+ if event and re.match('[a-z0-9]{10,}$', event):
+ body = '''I would like to join this thread: https://www.reddit.com/live/{1}
+
+(If you send this message with the following line intact, you will be invited automatically if possible)
+
+/autojoin service {0} event {1}'''.format(service_name, event)
+ elif event:
+ body = '''I would like to join this thread: {}'''.format(event)
+ else:
+ body = '''I would like to join this thread: (none)'''
+ assert re.match('[a-z0-9]+$', service_name)
+ qs = urllib.parse.urlencode({
+ 'to': '/r/livecounting',
+ 'subject': 'Please invite me',
+ 'message': body
+ })
+ return 'https://www.reddit.com/message/compose?{}'.format(qs)
+
+@app.route('/authorize')
+def authorize():
+ service_name = request.args.get('service', DEFAULT_SERVICE)
+ event = request.args.get('thread')
+ (cn, cr) = connect_db()
+ cr.execute("SELECT name, client_id, event_flake, redirect_uri FROM live_autojoin_allowed_event JOIN live_autojoin_service ON name = service_name JOIN reddit_app USING (client_id) WHERE service_name = %s AND event_flake = %s", (service_name, event))
+ try:
+ [(service_name, client_id, event, redirect_uri)] = cr.fetchall()
+ except ValueError:
+ privatemessage_url = make_privatemessage_url(service_name, event)
+ return render_template("error.html", privatemessage_url=privatemessage_url)
+ else:
+ return redirect(make_oauth_url(service_name, client_id, event, redirect_uri), code=303)
+ finally:
+ cn.close()
+
+@app.route('/invite')
+def invite():
+ authorization_code = request.args.get('code')
+ state = request.args.get('state')
+ (service_name, event_flake) = state.split(':')
+ ticket = secrets.token_urlsafe()
+ (cn, cr) = connect_db()
+ cr.execute("BEGIN")
+ cr.execute("WITH q1 AS (INSERT INTO reddit_app_authorization_code (state, code) VALUES (%s, %s) RETURNING id) INSERT INTO live_autojoin_ticket (service_name, ticket, issued_at, event_flake, authorization_code_id, status) SELECT %s, %s, CURRENT_TIMESTAMP, %s, id, NULL FROM q1", (state, authorization_code, service_name, ticket, event_flake))
+ cr.execute("NOTIFY live_autojoin")
+ cn.commit()
+ cn.close()
+ return redirect(url_for('ticket', service=service_name, ticket=ticket), code=303)
+
+@app.route('/ticket')
+def ticket():
+ service_name = request.args.get('service')
+ ticket = request.args.get('ticket')
+ (cn, cr) = connect_db()
+ cr.execute("SELECT event_flake, status FROM live_autojoin_ticket WHERE service_name = %s AND ticket = %s", (service_name, ticket))
+ try:
+ [(event, status)] = cr.fetchall()
+ except ValueError:
+ event = None
+ status = 'error'
+ cn.close()
+ if status in ('ok', 'already_contributor'):
+ return redirect('https://www.reddit.com/live/{}'.format(event), code=308)
+ elif status == 'error':
+ privatemessage_url = make_privatemessage_url(service_name, event)
+ return render_template("error.html", privatemessage_url=privatemessage_url)
+ else:
+ privatemessage_url = make_privatemessage_url(service_name, event)
+ return render_template("status.html", privatemessage_url=privatemessage_url)
+
+# TODO
+# - unallowed thread
+# - allowed but disabled thread
+# - authorization denied
+# - indication of progress/errors
+
+#- exchange authentication codes
+#- fetch authorization identity to populate ticket username
+#- invite users
+#- accept invites
--- /dev/null
+#!/bin/bash
+
+VERS=0.1.0
+
+python3 -m build --no-isolation
+tar c dist/sidebot-$VERS-py3-none-any.whl | ssh anders@bingus.internet6.net. tar x --strip-components=1 \&\& pip3 install --no-deps ./sidebot-$VERS-py3-none-any.whl
--- /dev/null
+[metadata]
+name = sidebot
+version = 0.1.0
+
+[options]
+packages = find:
+install_requires =
+ psycopg2 ~= 2.8
+include_package_data = True
+
+[options.package_data]
+sidebot =
+ notd_list
+
+[options.entry_points]
+console_scripts =
+ sidebot = sidebot.main:main
--- /dev/null
+import psycopg2
+import urllib.request
+import json
+import itertools
+from argparse import ArgumentParser
+from datetime import datetime, timezone
+from uuid import UUID
+
+from .number import number_from_update, is_kget, is_notd
+from .sidebar import find_parse_section, notd_banner_patch
+
+def main():
+
+ # retrieve run info
+
+ parser = ArgumentParser()
+ parser.add_argument('service_name')
+ args = parser.parse_args()
+
+ USER_AGENT='sidebot/0.1.0'
+ MAX_PAGES=3
+
+ dbconn = psycopg2.connect('')
+ db = dbconn.cursor()
+ db.execute('''
+ SELECT
+ service_name, event_id, last_update_id,
+ CURRENT_TIMESTAMP-COALESCE(freeze_age, '5min'),
+ COALESCE(keep_kget, 10),
+ COALESCE(keep_notd, 10),
+ access_token
+ FROM sidebot_service t0
+ JOIN reddit_app_authorization t1 ON t0.authorization_id=t1.id
+ WHERE service_name = %s AND CURRENT_TIMESTAMP < expires
+ FOR NO KEY UPDATE OF t0
+ ''', (args.service_name,))
+ [ (service_name, event_id, last_update_id, freeze_after, keep_kget, keep_notd, access_token) ] = db.fetchall()
+ last_update_dirty = False
+
+ # walk unread updates
+ updates = []
+ stopped_short = False
+ before_arg = "LiveUpdate_{}".format(last_update_id) if last_update_id else ""
+ for page_no in range(MAX_PAGES):
+ res = urllib.request.urlopen(urllib.request.Request(
+ 'https://www.reddit.com/live/{}.json?raw_json=1&limit=100&before={}'.format(
+ event_id, before_arg
+ ), headers={"User-Agent": USER_AGENT}))
+ if res.status != 200:
+ raise RuntimeError('HTTP {} {}'.format(res.status, res.reason))
+ page = json.load(res)['data']['children']
+ updates.extend((i['data'] for i in reversed(page)))
+ if before_arg != "" and len(page) > 0:
+ before_arg = page[0]['data']['name']
+ else:
+ break
+ else:
+ stopped_short = True
+
+ # update sidebar
+ res = urllib.request.urlopen(urllib.request.Request(
+ 'https://oauth.reddit.com/live/{}/about?raw_json=1'.format(event_id),
+ headers={"Authorization": "Bearer {}".format(access_token),
+ "User-Agent": USER_AGENT}))
+ if res.status != 200:
+ raise RuntimeError('HTTP {} {}'.format(res.status, res.reason))
+ data = json.load(res)['data']
+ resources = data['resources']
+ nsfw_arg = str(int(data['nsfw']))
+
+ (_, kget_slice, kget) = find_parse_section(resources, 'GET',
+ event_id=event_id, retain=keep_kget)
+ (notd_pos, notd_slice, notd) = find_parse_section(resources, 'Number of the Day',
+ event_id=event_id, retain=keep_notd)
+ if len(updates) > 0 and not stopped_short:
+ now = datetime.fromtimestamp(updates[-1]['created_utc'], tz=timezone.utc)
+ (notd_banner_slice, notd_banner) = notd_banner_patch(resources, notd_pos, now)
+
+ now_ts = None
+ time1 = UUID(last_update_id).time if last_update_id is not None else None
+ for u in updates:
+ ts = datetime.fromtimestamp(u['created_utc'], tz=timezone.utc)
+ u_uuid = UUID(u['id'])
+ if ts < freeze_after:
+ last_update_id = u['id']
+ last_update_dirty = True
+ if now_ts is None or ts > now_ts:
+ now_ts = ts
+
+ # delete lines for missing (id between previous and current update) (hence deleted) updates
+ if time1 is not None:
+ for processor in [kget, notd]:
+ processor.delete_between(time1, u_uuid.time)
+ time1 = u_uuid.time
+
+ n = number_from_update(u)
+ delete_matching = u['stricken']
+ for (processor, is_special) in [(kget, is_kget), (notd, is_notd)]:
+ found_matching = processor.take(u_uuid, delete_matching)
+ if n is not None and is_special(ts, n) and not u['stricken'] and not found_matching:
+ processor.insert(u, n)
+
+ def patch_text(text, patches):
+ for (slice_, new_text) in sorted(patches, key=lambda patch: -patch[0].start):
+ text = text[:slice_.start] + new_text + text[slice_.stop:]
+ return text
+
+ new_resources = patch_text(resources, itertools.chain((
+ (slice_, proc.text()) for (slice_, proc) in [
+ (kget_slice, kget),
+ (notd_slice, notd)
+ ] if proc.dirty),
+ ((notd_banner_slice, notd_banner) for d in range(int(not stopped_short and now_ts is not None)))
+ ))
+
+ if new_resources != resources:
+ req = urllib.request.Request(
+ 'https://oauth.reddit.com/live/{}/edit'.format(event_id),
+ data=urllib.parse.urlencode({
+ 'api_type': 'json',
+ 'title': data['title'],
+ 'description': data['description'],
+ 'resources': new_resources,
+ 'nsfw': str(int(data['nsfw']))
+ }).encode(),
+ headers={
+ 'User-Agent': USER_AGENT,
+ 'Authorization': 'Bearer {}'.format(access_token)
+ },
+ method='POST'
+ )
+ res = urllib.request.urlopen(req)
+ if res.status != 200:
+ raise RuntimeError('HTTP {} {}'.format(res.status, res.reason))
+
+ # update run info
+ if last_update_dirty:
+ db.execute('''
+ UPDATE sidebot_service SET last_update_id=%s WHERE service_name=%s
+ ''', (last_update_id, service_name))
+ dbconn.commit()
--- /dev/null
+2022-03-24 555\r
+2022-03-25 792\r
+2022-03-26 961\r
+2022-03-27 395\r
+2022-03-28 813\r
+2022-03-29 489\r
+2022-03-30 861\r
+2022-03-31 243\r
+2022-04-01 700\r
+2022-04-02 320\r
+2022-04-03 635\r
+2022-04-04 494\r
+2022-04-05 666\r
+2022-04-06 976\r
+2022-04-07 288\r
+2022-04-08 298\r
+2022-04-09 444\r
+2022-04-10 650\r
+2022-04-11 667\r
+2022-04-12 921\r
+2022-04-13 279\r
+2022-04-14 825\r
+2022-04-15 718\r
+2022-04-16 274\r
+2022-04-17 623\r
+2022-04-18 695\r
+2022-04-19 355\r
+2022-04-20 341\r
+2022-04-21 751\r
+2022-04-22 112\r
+2022-04-23 050\r
+2022-04-24 280\r
+2022-04-25 628\r
+2022-04-26 477\r
+2022-04-27 540\r
+2022-04-28 629\r
+2022-04-29 070\r
+2022-04-30 954\r
+2022-05-01 411\r
+2022-05-02 008\r
+2022-05-03 184\r
+2022-05-04 219\r
+2022-05-05 412\r
+2022-05-06 450\r
+2022-05-07 396\r
+2022-05-08 701\r
+2022-05-09 716\r
+2022-05-10 442\r
+2022-05-11 619\r
+2022-05-12 648\r
+2022-05-13 625\r
+2022-05-14 196\r
+2022-05-15 518\r
+2022-05-16 443\r
+2022-05-17 574\r
+2022-05-18 359\r
+2022-05-19 855\r
+2022-05-20 539\r
+2022-05-21 257\r
+2022-05-22 194\r
+2022-05-23 586\r
+2022-05-24 264\r
+2022-05-25 333\r
+2022-05-26 820\r
+2022-05-27 462\r
+2022-05-28 174\r
+2022-05-29 440\r
+2022-05-30 466\r
+2022-05-31 431\r
+2022-06-01 111\r
+2022-06-02 513\r
+2022-06-03 672\r
+2022-06-04 641\r
+2022-06-05 381\r
+2022-06-06 269\r
+2022-06-07 758\r
+2022-06-08 679\r
+2022-06-09 979\r
+2022-06-10 297\r
+2022-06-11 517\r
+2022-06-12 527\r
+2022-06-13 521\r
+2022-06-14 088\r
+2022-06-15 091\r
+2022-06-16 575\r
+2022-06-17 844\r
+2022-06-18 707\r
+2022-06-19 636\r
+2022-06-20 734\r
+2022-06-21 334\r
+2022-06-22 329\r
+2022-06-23 886\r
+2022-06-24 469\r
+2022-06-25 725\r
+2022-06-26 158\r
+2022-06-27 225\r
+2022-06-28 831\r
+2022-06-29 958\r
+2022-06-30 832\r
+2022-07-01 991\r
+2022-07-02 508\r
+2022-07-03 993\r
+2022-07-04 035\r
+2022-07-05 382\r
+2022-07-06 027\r
+2022-07-07 594\r
+2022-07-08 083\r
+2022-07-09 053\r
+2022-07-10 046\r
+2022-07-11 694\r
+2022-07-12 816\r
+2022-07-13 840\r
+2022-07-14 255\r
+2022-07-15 018\r
+2022-07-16 137\r
+2022-07-17 901\r
+2022-07-18 572\r
+2022-07-19 276\r
+2022-07-20 332\r
+2022-07-21 163\r
+2022-07-22 305\r
+2022-07-23 726\r
+2022-07-24 826\r
+2022-07-25 026\r
+2022-07-26 780\r
+2022-07-27 585\r
+2022-07-28 670\r
+2022-07-29 994\r
+2022-07-30 793\r
+2022-07-31 785\r
+2022-08-01 364\r
+2022-08-02 788\r
+2022-08-03 801\r
+2022-08-04 267\r
+2022-08-05 419\r
+2022-08-06 982\r
+2022-08-07 913\r
+2022-08-08 974\r
+2022-08-09 087\r
+2022-08-10 804\r
+2022-08-11 658\r
+2022-08-12 080\r
+2022-08-13 189\r
+2022-08-14 709\r
+2022-08-15 497\r
+2022-08-16 475\r
+2022-08-17 148\r
+2022-08-18 213\r
+2022-08-19 795\r
+2022-08-20 875\r
+2022-08-21 438\r
+2022-08-22 817\r
+2022-08-23 537\r
+2022-08-24 222\r
+2022-08-25 806\r
+2022-08-26 100\r
+2022-08-27 676\r
+2022-08-28 304\r
+2022-08-29 871\r
+2022-08-30 217\r
+2022-08-31 535\r
+2022-09-01 953\r
+2022-09-02 786\r
+2022-09-03 435\r
+2022-09-04 453\r
+2022-09-05 653\r
+2022-09-06 891\r
+2022-09-07 480\r
+2022-09-08 430\r
+2022-09-09 834\r
+2022-09-10 065\r
+2022-09-11 919\r
+2022-09-12 766\r
+2022-09-13 328\r
+2022-09-14 884\r
+2022-09-15 128\r
+2022-09-16 192\r
+2022-09-17 946\r
+2022-09-18 922\r
+2022-09-19 183\r
+2022-09-20 424\r
+2022-09-21 032\r
+2022-09-22 810\r
+2022-09-23 230\r
+2022-09-24 290\r
+2022-09-25 479\r
+2022-09-26 673\r
+2022-09-27 730\r
+2022-09-28 086\r
+2022-09-29 017\r
+2022-09-30 374\r
+2022-10-01 544\r
+2022-10-02 114\r
+2022-10-03 187\r
+2022-10-04 596\r
+2022-10-05 167\r
+2022-10-06 638\r
+2022-10-07 038\r
+2022-10-08 319\r
+2022-10-09 405\r
+2022-10-10 295\r
+2022-10-11 761\r
+2022-10-12 249\r
+2022-10-13 573\r
+2022-10-14 242\r
+2022-10-15 084\r
+2022-10-16 503\r
+2022-10-17 584\r
+2022-10-18 905\r
+2022-10-19 546\r
+2022-10-20 975\r
+2022-10-21 110\r
+2022-10-22 342\r
+2022-10-23 511\r
+2022-10-24 851\r
+2022-10-25 433\r
+2022-10-26 471\r
+2022-10-27 669\r
+2022-10-28 291\r
+2022-10-29 486\r
+2022-10-30 119\r
+2022-10-31 784\r
+2022-11-01 872\r
+2022-11-02 985\r
+2022-11-03 956\r
+2022-11-04 208\r
+2022-11-05 779\r
+2022-11-06 727\r
+2022-11-07 887\r
+2022-11-08 275\r
+2022-11-09 598\r
+2022-11-10 129\r
+2022-11-11 987\r
+2022-11-12 971\r
+2022-11-13 615\r
+2022-11-14 782\r
+2022-11-15 942\r
+2022-11-16 245\r
+2022-11-17 246\r
+2022-11-18 069\r
+2022-11-19 476\r
+2022-11-20 002\r
+2022-11-21 862\r
+2022-11-22 240\r
+2022-11-23 244\r
+2022-11-24 538\r
+2022-11-25 507\r
+2022-11-26 337\r
+2022-11-27 409\r
+2022-11-28 007\r
+2022-11-29 024\r
+2022-11-30 755\r
+2022-12-01 452\r
+2022-12-02 918\r
+2022-12-03 202\r
+2022-12-04 455\r
+2022-12-05 340\r
+2022-12-06 859\r
+2022-12-07 347\r
+2022-12-08 743\r
+2022-12-09 948\r
+2022-12-10 717\r
+2022-12-11 547\r
+2022-12-12 323\r
+2022-12-13 048\r
+2022-12-14 533\r
+2022-12-15 072\r
+2022-12-16 947\r
+2022-12-17 140\r
+2022-12-18 098\r
+2022-12-19 301\r
+2022-12-20 270\r
+2022-12-21 601\r
+2022-12-22 759\r
+2022-12-23 089\r
+2022-12-24 380\r
+2022-12-25 819\r
+2022-12-26 552\r
+2022-12-27 343\r
+2022-12-28 776\r
+2022-12-29 889\r
+2022-12-30 226\r
+2022-12-31 990\r
+2023-01-01 212\r
+2023-01-02 314\r
+2023-01-03 560\r
+2023-01-04 420\r
+2023-01-05 767\r
+2023-01-06 746\r
+2023-01-07 200\r
+2023-01-08 915\r
+2023-01-09 595\r
+2023-01-10 231\r
+2023-01-11 739\r
+2023-01-12 451\r
+2023-01-13 803\r
+2023-01-14 529\r
+2023-01-15 131\r
+2023-01-16 928\r
+2023-01-17 259\r
+2023-01-18 618\r
+2023-01-19 556\r
+2023-01-20 456\r
+2023-01-21 447\r
+2023-01-22 603\r
+2023-01-23 113\r
+2023-01-24 980\r
+2023-01-25 876\r
+2023-01-26 853\r
+2023-01-27 418\r
+2023-01-28 400\r
+2023-01-29 185\r
+2023-01-30 704\r
+2023-01-31 460\r
+2023-02-01 012\r
+2023-02-02 481\r
+2023-02-03 737\r
+2023-02-04 655\r
+2023-02-05 504\r
+2023-02-06 223\r
+2023-02-07 870\r
+2023-02-08 056\r
+2023-02-09 723\r
+2023-02-10 156\r
+2023-02-11 744\r
+2023-02-12 317\r
+2023-02-13 828\r
+2023-02-14 016\r
+2023-02-15 668\r
+2023-02-16 577\r
+2023-02-17 311\r
+2023-02-18 798\r
+2023-02-19 763\r
+2023-02-20 756\r
+2023-02-21 797\r
+2023-02-22 258\r
+2023-02-23 986\r
+2023-02-24 932\r
+2023-02-25 461\r
+2023-02-26 608\r
+2023-02-27 662\r
+2023-02-28 711\r
+2023-03-01 394\r
+2023-03-02 765\r
+2023-03-03 195\r
+2023-03-04 892\r
+2023-03-05 238\r
+2023-03-06 881\r
+2023-03-07 044\r
+2023-03-08 534\r
+2023-03-09 833\r
+2023-03-10 729\r
+2023-03-11 353\r
+2023-03-12 849\r
+2023-03-13 250\r
+2023-03-14 188\r
+2023-03-15 375\r
+2023-03-16 660\r
+2023-03-17 108\r
+2023-03-18 972\r
+2023-03-19 182\r
+2023-03-20 748\r
+2023-03-21 706\r
+2023-03-22 549\r
+2023-03-23 292\r
+2023-03-24 807\r
+2023-03-25 097\r
+2023-03-26 885\r
+2023-03-27 437\r
+2023-03-28 068\r
+2023-03-29 519\r
+2023-03-30 271\r
+2023-03-31 789\r
+2023-04-01 005\r
+2023-04-02 663\r
+2023-04-03 416\r
+2023-04-04 022\r
+2023-04-05 345\r
+2023-04-06 176\r
+2023-04-07 703\r
+2023-04-08 883\r
+2023-04-09 335\r
+2023-04-10 970\r
+2023-04-11 952\r
+2023-04-12 877\r
+2023-04-13 620\r
+2023-04-14 583\r
+2023-04-15 105\r
+2023-04-16 551\r
+2023-04-17 423\r
+2023-04-18 251\r
+2023-04-19 141\r
+2023-04-20 029\r
+2023-04-21 714\r
+2023-04-22 142\r
+2023-04-23 150\r
+2023-04-24 652\r
+2023-04-25 265\r
+2023-04-26 241\r
+2023-04-27 847\r
+2023-04-28 175\r
+2023-04-29 324\r
+2023-04-30 929\r
+2023-05-01 488\r
+2023-05-02 562\r
+2023-05-03 863\r
+2023-05-04 161\r
+2023-05-05 617\r
+2023-05-06 367\r
+2023-05-07 428\r
+2023-05-08 310\r
+2023-05-09 988\r
+2023-05-10 095\r
+2023-05-11 307\r
+2023-05-12 391\r
+2023-05-13 499\r
+2023-05-14 908\r
+2023-05-15 741\r
+2023-05-16 385\r
+2023-05-17 649\r
+2023-05-18 532\r
+2023-05-19 357\r
+2023-05-20 039\r
+2023-05-21 025\r
+2023-05-22 286\r
+2023-05-23 413\r
+2023-05-24 710\r
+2023-05-25 515\r
+2023-05-26 642\r
+2023-05-27 911\r
+2023-05-28 774\r
+2023-05-29 712\r
+2023-05-30 033\r
+2023-05-31 346\r
+2023-06-01 253\r
+2023-06-02 647\r
+2023-06-03 331\r
+2023-06-04 530\r
+2023-06-05 211\r
+2023-06-06 651\r
+2023-06-07 610\r
+2023-06-08 322\r
+2023-06-09 151\r
+2023-06-10 548\r
+2023-06-11 554\r
+2023-06-12 392\r
+2023-06-13 256\r
+2023-06-14 308\r
+2023-06-15 296\r
+2023-06-16 389\r
+2023-06-17 815\r
+2023-06-18 454\r
+2023-06-19 229\r
+2023-06-20 550\r
+2023-06-21 309\r
+2023-06-22 664\r
+2023-06-23 582\r
+2023-06-24 754\r
+2023-06-25 722\r
+2023-06-26 597\r
+2023-06-27 160\r
+2023-06-28 436\r
+2023-06-29 827\r
+2023-06-30 580\r
+2023-07-01 277\r
+2023-07-02 028\r
+2023-07-03 802\r
+2023-07-04 387\r
+2023-07-05 692\r
+2023-07-06 313\r
+2023-07-07 397\r
+2023-07-08 731\r
+2023-07-09 272\r
+2023-07-10 950\r
+2023-07-11 432\r
+2023-07-12 643\r
+2023-07-13 015\r
+2023-07-14 837\r
+2023-07-15 589\r
+2023-07-16 568\r
+2023-07-17 674\r
+2023-07-18 293\r
+2023-07-19 943\r
+2023-07-20 688\r
+2023-07-21 567\r
+2023-07-22 852\r
+2023-07-23 143\r
+2023-07-24 613\r
+2023-07-25 014\r
+2023-07-26 144\r
+2023-07-27 978\r
+2023-07-28 682\r
+2023-07-29 120\r
+2023-07-30 093\r
+2023-07-31 149\r
+2023-08-01 675\r
+2023-08-02 349\r
+2023-08-03 559\r
+2023-08-04 907\r
+2023-08-05 122\r
+2023-08-06 448\r
+2023-08-07 482\r
+2023-08-08 422\r
+2023-08-09 772\r
+2023-08-10 073\r
+2023-08-11 566\r
+2023-08-12 262\r
+2023-08-13 906\r
+2023-08-14 955\r
+2023-08-15 326\r
+2023-08-16 204\r
+2023-08-17 936\r
+2023-08-18 449\r
+2023-08-19 637\r
+2023-08-20 441\r
+2023-08-21 360\r
+2023-08-22 376\r
+2023-08-23 811\r
+2023-08-24 964\r
+2023-08-25 198\r
+2023-08-26 165\r
+2023-08-27 843\r
+2023-08-28 414\r
+2023-08-29 096\r
+2023-08-30 775\r
+2023-08-31 627\r
+2023-09-01 894\r
+2023-09-02 984\r
+2023-09-03 516\r
+2023-09-04 366\r
+2023-09-05 306\r
+2023-09-06 611\r
+2023-09-07 808\r
+2023-09-08 171\r
+2023-09-09 078\r
+2023-09-10 657\r
+2023-09-11 030\r
+2023-09-12 458\r
+2023-09-13 752\r
+2023-09-14 094\r
+2023-09-15 867\r
+2023-09-16 967\r
+2023-09-17 439\r
+2023-09-18 850\r
+2023-09-19 646\r
+2023-09-20 159\r
+2023-09-21 879\r
+2023-09-22 733\r
+2023-09-23 484\r
+2023-09-24 895\r
+2023-09-25 888\r
+2023-09-26 924\r
+2023-09-27 338\r
+2023-09-28 383\r
+2023-09-29 402\r
+2023-09-30 047\r
+2023-10-01 545\r
+2023-10-02 690\r
+2023-10-03 916\r
+2023-10-04 732\r
+2023-10-05 133\r
+2023-10-06 805\r
+2023-10-07 599\r
+2023-10-08 483\r
+2023-10-09 075\r
+2023-10-10 912\r
+2023-10-11 794\r
+2023-10-12 247\r
+2023-10-13 790\r
+2023-10-14 591\r
+2023-10-15 157\r
+2023-10-16 145\r
+2023-10-17 472\r
+2023-10-18 553\r
+2023-10-19 873\r
+2023-10-20 983\r
+2023-10-21 864\r
+2023-10-22 294\r
+2023-10-23 074\r
+2023-10-24 268\r
+2023-10-25 634\r
+2023-10-26 630\r
+2023-10-27 130\r
+2023-10-28 235\r
+2023-10-29 565\r
+2023-10-30 285\r
+2023-10-31 934\r
+2023-11-01 791\r
+2023-11-02 136\r
+2023-11-03 777\r
+2023-11-04 287\r
+2023-11-05 283\r
+2023-11-06 059\r
+2023-11-07 071\r
+2023-11-08 940\r
+2023-11-09 004\r
+2023-11-10 702\r
+2023-11-11 368\r
+2023-11-12 512\r
+2023-11-13 624\r
+2023-11-14 421\r
+2023-11-15 118\r
+2023-11-16 680\r
+2023-11-17 273\r
+2023-11-18 216\r
+2023-11-19 957\r
+2023-11-20 254\r
+2023-11-21 115\r
+2023-11-22 762\r
+2023-11-23 052\r
+2023-11-24 944\r
+2023-11-25 882\r
+2023-11-26 814\r
+2023-11-27 210\r
+2023-11-28 926\r
+2023-11-29 197\r
+2023-11-30 939\r
+2023-12-01 369\r
+2023-12-02 931\r
+2023-12-03 600\r
+2023-12-04 498\r
+2023-12-05 686\r
+2023-12-06 478\r
+2023-12-07 180\r
+2023-12-08 485\r
+2023-12-09 303\r
+2023-12-10 633\r
+2023-12-11 683\r
+2023-12-12 514\r
+2023-12-13 013\r
+2023-12-14 868\r
+2023-12-15 848\r
+2023-12-16 614\r
+2023-12-17 201\r
+2023-12-18 536\r
+2023-12-19 681\r
+2023-12-20 866\r
+2023-12-21 671\r
+2023-12-22 066\r
+2023-12-23 186\r
+2023-12-24 233\r
+2023-12-25 104\r
+2023-12-26 404\r
+2023-12-27 773\r
+2023-12-28 281\r
+2023-12-29 099\r
+2023-12-30 570\r
+2023-12-31 632\r
+2024-01-01 152\r
+2024-01-02 146\r
+2024-01-03 371\r
+2024-01-04 969\r
+2024-01-05 330\r
+2024-01-06 992\r
+2024-01-07 740\r
+2024-01-08 316\r
+2024-01-09 968\r
+2024-01-10 757\r
+2024-01-11 058\r
+2024-01-12 429\r
+2024-01-13 178\r
+2024-01-14 787\r
+2024-01-15 252\r
+2024-01-16 812\r
+2024-01-17 106\r
+2024-01-18 325\r
+2024-01-19 631\r
+2024-01-20 576\r
+2024-01-21 147\r
+2024-01-22 132\r
+2024-01-23 434\r
+2024-01-24 490\r
+2024-01-25 220\r
+2024-01-26 778\r
+2024-01-27 164\r
+2024-01-28 989\r
+2024-01-29 661\r
+2024-01-30 135\r
+2024-01-31 645\r
+2024-02-01 036\r
+2024-02-02 169\r
+2024-02-03 665\r
+2024-02-04 037\r
+2024-02-05 590\r
+2024-02-06 914\r
+2024-02-07 836\r
+2024-02-08 172\r
+2024-02-09 770\r
+2024-02-10 049\r
+2024-02-11 581\r
+2024-02-12 067\r
+2024-02-13 639\r
+2024-02-14 386\r
+2024-02-15 903\r
+2024-02-16 398\r
+2024-02-17 232\r
+2024-02-18 966\r
+2024-02-19 525\r
+2024-02-20 393\r
+2024-02-21 261\r
+2024-02-22 728\r
+2024-02-23 101\r
+2024-02-24 557\r
+2024-02-25 896\r
+2024-02-26 057\r
+2024-02-27 962\r
+2024-02-28 406\r
+2024-02-29 399\r
+2024-03-01 998\r
+2024-03-02 352\r
+2024-03-03 509\r
+2024-03-04 500\r
+2024-03-05 378\r
+2024-03-06 365\r
+2024-03-07 520\r
+2024-03-08 496\r
+2024-03-09 354\r
+2024-03-10 457\r
+2024-03-11 205\r
+2024-03-12 605\r
+2024-03-13 705\r
+2024-03-14 010\r
+2024-03-15 897\r
+2024-03-16 588\r
+2024-03-17 055\r
+2024-03-18 749\r
+2024-03-19 459\r
+2024-03-20 363\r
+2024-03-21 745\r
+2024-03-22 742\r
+2024-03-23 656\r
+2024-03-24 818\r
+2024-03-25 684\r
+2024-03-26 999\r
+2024-03-27 302\r
+2024-03-28 612\r
+2024-03-29 841\r
+2024-03-30 041\r
+2024-03-31 880\r
+2024-04-01 042\r
+2024-04-02 464\r
+2024-04-03 965\r
+2024-04-04 846\r
+2024-04-05 203\r
+2024-04-06 856\r
+2024-04-07 693\r
+2024-04-08 190\r
+2024-04-09 505\r
+2024-04-10 103\r
+2024-04-11 842\r
+2024-04-12 691\r
+2024-04-13 154\r
+2024-04-14 239\r
+2024-04-15 351\r
+2024-04-16 644\r
+2024-04-17 051\r
+2024-04-18 738\r
+2024-04-19 116\r
+2024-04-20 206\r
+2024-04-21 379\r
+2024-04-22 278\r
+2024-04-23 720\r
+2024-04-24 318\r
+2024-04-25 699\r
+2024-04-26 900\r
+2024-04-27 234\r
+2024-04-28 109\r
+2024-04-29 510\r
+2024-04-30 043\r
+2024-05-01 237\r
+2024-05-02 289\r
+2024-05-03 224\r
+2024-05-04 063\r
+2024-05-05 747\r
+2024-05-06 769\r
+2024-05-07 685\r
+2024-05-08 470\r
+2024-05-09 824\r
+2024-05-10 177\r
+2024-05-11 937\r
+2024-05-12 218\r
+2024-05-13 491\r
+2024-05-14 949\r
+2024-05-15 082\r
+2024-05-16 951\r
+2024-05-17 260\r
+2024-05-18 621\r
+2024-05-19 973\r
+2024-05-20 467\r
+2024-05-21 874\r
+2024-05-22 687\r
+2024-05-23 941\r
+2024-05-24 121\r
+2024-05-25 336\r
+2024-05-26 531\r
+2024-05-27 401\r
+2024-05-28 781\r
+2024-05-29 023\r
+2024-05-30 079\r
+2024-05-31 981\r
+2024-06-01 417\r
+2024-06-02 721\r
+2024-06-03 060\r
+2024-06-04 181\r
+2024-06-05 902\r
+2024-06-06 838\r
+2024-06-07 001\r
+2024-06-08 350\r
+2024-06-09 327\r
+2024-06-10 474\r
+2024-06-11 248\r
+2024-06-12 910\r
+2024-06-13 372\r
+2024-06-14 125\r
+2024-06-15 124\r
+2024-06-16 963\r
+2024-06-17 750\r
+2024-06-18 558\r
+2024-06-19 977\r
+2024-06-20 996\r
+2024-06-21 602\r
+2024-06-22 228\r
+2024-06-23 373\r
+2024-06-24 117\r
+2024-06-25 339\r
+2024-06-26 312\r
+2024-06-27 321\r
+2024-06-28 835\r
+2024-06-29 592\r
+2024-06-30 854\r
+2024-07-01 492\r
+2024-07-02 654\r
+2024-07-03 821\r
+2024-07-04 214\r
+2024-07-05 959\r
+2024-07-06 904\r
+2024-07-07 227\r
+2024-07-08 045\r
+2024-07-09 139\r
+2024-07-10 502\r
+2024-07-11 031\r
+2024-07-12 487\r
+2024-07-13 561\r
+2024-07-14 377\r
+2024-07-15 593\r
+2024-07-16 344\r
+2024-07-17 563\r
+2024-07-18 266\r
+2024-07-19 076\r
+2024-07-20 425\r
+2024-07-21 123\r
+2024-07-22 522\r
+2024-07-23 564\r
+2024-07-24 506\r
+2024-07-25 040\r
+2024-07-26 526\r
+2024-07-27 034\r
+2024-07-28 021\r
+2024-07-29 207\r
+2024-07-30 768\r
+2024-07-31 179\r
+2024-08-01 426\r
+2024-08-02 236\r
+2024-08-03 809\r
+2024-08-04 199\r
+2024-08-05 933\r
+2024-08-06 020\r
+2024-08-07 909\r
+2024-08-08 995\r
+2024-08-09 736\r
+2024-08-10 640\r
+2024-08-11 390\r
+2024-08-12 011\r
+2024-08-13 542\r
+2024-08-14 299\r
+2024-08-15 446\r
+2024-08-16 893\r
+2024-08-17 713\r
+2024-08-18 501\r
+2024-08-19 410\r
+2024-08-20 282\r
+2024-08-21 468\r
+2024-08-22 362\r
+2024-08-23 917\r
+2024-08-24 899\r
+2024-08-25 000\r
+2024-08-26 997\r
+2024-08-27 783\r
+2024-08-28 407\r
+2024-08-29 708\r
+2024-08-30 493\r
+2024-08-31 019\r
+2024-09-01 869\r
+2024-09-02 898\r
+2024-09-03 062\r
+2024-09-04 845\r
+2024-09-05 760\r
+2024-09-06 604\r
+2024-09-07 923\r
+2024-09-08 162\r
+2024-09-09 127\r
+2024-09-10 090\r
+2024-09-11 607\r
+2024-09-12 626\r
+2024-09-13 865\r
+2024-09-14 215\r
+2024-09-15 753\r
+2024-09-16 616\r
+2024-09-17 822\r
+2024-09-18 945\r
+2024-09-19 858\r
+2024-09-20 698\r
+2024-09-21 771\r
+2024-09-22 085\r
+2024-09-23 445\r
+2024-09-24 609\r
+2024-09-25 081\r
+2024-09-26 829\r
+2024-09-27 300\r
+2024-09-28 193\r
+2024-09-29 960\r
+2024-09-30 092\r
+2024-10-01 541\r
+2024-10-02 356\r
+2024-10-03 659\r
+2024-10-04 697\r
+2024-10-05 796\r
+2024-10-06 107\r
+2024-10-07 403\r
+2024-10-08 719\r
+2024-10-09 696\r
+2024-10-10 677\r
+2024-10-11 878\r
+2024-10-12 284\r
+2024-10-13 388\r
+2024-10-14 927\r
+2024-10-15 569\r
+2024-10-16 724\r
+2024-10-17 930\r
+2024-10-18 054\r
+2024-10-19 935\r
+2024-10-20 361\r
+2024-10-21 221\r
+2024-10-22 465\r
+2024-10-23 622\r
+2024-10-24 170\r
+2024-10-25 839\r
+2024-10-26 415\r
+2024-10-27 427\r
+2024-10-28 077\r
+2024-10-29 370\r
+2024-10-30 587\r
+2024-10-31 925\r
+2024-11-01 134\r
+2024-11-02 315\r
+2024-11-03 571\r
+2024-11-04 735\r
+2024-11-05 191\r
+2024-11-06 408\r
+2024-11-07 006\r
+2024-11-08 166\r
+2024-11-09 579\r
+2024-11-10 764\r
+2024-11-11 543\r
+2024-11-12 823\r
+2024-11-13 102\r
+2024-11-14 800\r
+2024-11-15 857\r
+2024-11-16 168\r
+2024-11-17 473\r
+2024-11-18 348\r
+2024-11-19 890\r
+2024-11-20 209\r
+2024-11-21 920\r
+2024-11-22 064\r
+2024-11-23 578\r
+2024-11-24 263\r
+2024-11-25 606\r
+2024-11-26 009\r
+2024-11-27 938\r
+2024-11-28 524\r
+2024-11-29 126\r
+2024-11-30 061\r
+2024-12-01 799\r
+2024-12-02 463\r
+2024-12-03 830\r
+2024-12-04 689\r
+2024-12-05 003\r
+2024-12-06 860\r
+2024-12-07 155\r
+2024-12-08 715\r
+2024-12-09 523\r
+2024-12-10 678\r
+2024-12-11 384\r
+2024-12-12 528\r
+2024-12-13 173\r
+2024-12-14 138\r
+2024-12-15 153\r
+2024-12-16 358\r
+2024-12-17 495
\ No newline at end of file
--- /dev/null
+from importlib import resources
+import csv
+from datetime import date
+import zoneinfo
+
+from .strikebot_updates import parse_update
+
+with resources.files('sidebot').joinpath('notd_list').open() as notd_file:
+ notd_list = { date.fromisoformat(row[0]): int(row[1]) for row in csv.reader(notd_file, delimiter='\t', quoting=csv.QUOTE_NONE) if row[0] }
+
+notd_tz = zoneinfo.ZoneInfo('US/Eastern')
+
+def number_from_update(u):
+ return parse_update(u, None, '').number
+
+def format_n(n):
+ return '{:,}'.format(n)
+
+def is_kget(ts, n):
+ return n > 0 and n % 1000 == 0
+
+def is_notd(ts, n):
+ if n <= 0:
+ return False
+ residue = n % 1000
+ ts_date = ts.astimezone(notd_tz).date()
+ return ts_date in notd_list and residue == int(notd_list[ts_date])
+
+def notd_banner(ts):
+ notd_key = ts.astimezone(notd_tz).date()
+ if notd_key in notd_list:
+ notd_str = '{:03d}'.format(notd_list[notd_key])
+ else:
+ notd_str = '???'
+ return '{}: XX,XXX,{}'.format(notd_key, notd_str)
--- /dev/null
+import re
+from datetime import datetime
+from uuid import UUID
+
+from .number import format_n, notd_banner
+
+def uuid_from_regular_line(event_id, l):
+ match = re.match(r'\* \[[1-9]\d{0,2}(?:,\d{3})*\]\(/live/([a-z0-9]+)/updates/([0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12})\) - /u/[0-9a-zA-Z_-]{3,20}', l)
+ if not match or match.group(1) != event_id:
+ return None
+ return UUID(match.group(2))
+
+def find_parse_section(text, section_name, **kwargs):
+ marker1 = '[](#sidebar "start {}")\n\n'.format(section_name)
+ marker2 = '[](#sidebar "end {}")'.format(section_name)
+
+ ix0 = text.index(marker1)
+ if ix0 == -1:
+ return None
+ ix1 = ix0 + len(marker1)
+ ix2 = text.index(marker2, ix1)
+
+ raw_lines = [] if ix1 == ix2 else text[ix1:ix2].split('\n')
+ line_processor = LineProcessor(reversed(raw_lines), **kwargs)
+ return (ix0, slice(ix1, ix2), line_processor)
+
+def notd_banner_patch(text, notd_marker1, now):
+ banner_line_end = text.rfind('\n', 0, notd_marker1)
+ if banner_line_end == -1: # marker line is first line; we'll insert
+ banner_line_end = 0
+ banner_line_begin = text.rfind('\n', 0, banner_line_end) + 1
+
+ new_text = notd_banner(now).replace('\n', '')
+ return (slice(banner_line_begin, banner_line_end), new_text)
+
+class LineProcessor:
+ def __init__(self, lines, event_id, retain=10):
+ self.event_id = event_id
+ self.lines = [(l, uuid_from_regular_line(event_id, l)) for l in lines]
+ self.i = 0
+ self.i_prev_uuid = None
+ self.j = -retain #?
+ self.dirty = False
+
+ def delete_between(self, time1, time2):
+ while not self.end():
+ if self.valid():
+ if time1 < self.time() < time2:
+ self.delete()
+ continue
+ elif self.time() >= time2:
+ break
+ self.next()
+
+ # advance through lines matching target_uuid, skipping invalid ones.
+ # delete matching lines if delete=True.
+ # return whether there were matching lines.
+ def take(self, target_uuid, delete_matching):
+ found_matching = False
+ while not self.end():
+ if self.valid():
+ if self.time() > target_uuid.time:
+ break
+ if self.uuid() == target_uuid:
+ found_matching = True
+ if delete_matching:
+ self.delete()
+ continue
+ self.next()
+ return found_matching
+
+ # Advance the i pointer. If advancing through a regular, valid, novel line then also
+ # advance the j pointer afterward.
+ def next(self):
+ advance_j = False
+ if self.valid():
+ if self.i_prev_uuid != self.uuid(): # novel
+ self.i_prev_uuid = self.uuid()
+ advance_j = True
+ self.i += 1
+ if advance_j:
+ self.next_j()
+
+ # Advance the j pointer by deleting one or more regular valid lines of a single id
+ def next_j(self):
+ if self.j < 0: # j behind start of list - no actual lines, nothing to delete
+ self.j += 1
+ return
+ assert self.j < self.i # we should not advance j without a line to look at
+ if self.j == 0: # starting out: skip over initial non-regular lines
+ while self.j < self.i and self.lines[self.j][1] is None:
+ self.j += 1
+ assert self.j < self.i # we should not advance j without a regular line to look at
+ cur_uuid = self.lines[self.j][1]
+ assert cur_uuid is not None
+ while self.j < self.i: # don't let j advance beyond i (proxy for end of list)
+ if (self.lines[self.j][1] is None # non-regular
+ or self.lines[self.j][1].time > cur_uuid.time): # invalid
+ self.j += 1 # skip over
+ else:
+ if self.lines[self.j][1] != cur_uuid: # novel, stop
+ break
+ # line at self.j is valid, regular, not novel: delete
+ del self.lines[self.j]
+ self.i -= 1
+ self.dirty = True
+
+ def insert(self, u, n):
+ str_ = '* [{}](/live/{}/updates/{}) - /u/{}'.format(
+ format_n(n), self.event_id, u['id'], u['author'])
+ self.lines.insert(self.i, (str_, UUID(u['id'])))
+ self.next()
+ self.dirty = True
+
+ def delete(self):
+ assert self.i < len(self.lines)
+ del self.lines[self.i]
+ self.dirty = True
+
+ def end(self):
+ return self.i == len(self.lines)
+
+ def uuid(self):
+ assert not self.end()
+ return self.lines[self.i][1]
+
+ def time(self):
+ assert not self.end()
+ return self.uuid().time
+
+ def valid(self):
+ assert not self.end()
+ if self.uuid() is None:
+ return False
+ return self.i_prev_uuid is None or self.time() >= self.i_prev_uuid.time
+
+ def text(self):
+ return '\n'.join((l[0] for l in reversed(self.lines)))
--- /dev/null
+# copied from strikebot/updates.py
+
+from __future__ import annotations
+from dataclasses import dataclass
+from enum import Enum
+from typing import Optional
+import re
+
+from bs4 import BeautifulSoup
+
+
+Command = Enum("Command", ["RESET", "REPORT"])
+
+
+@dataclass
+class ParsedUpdate:
+ number: Optional[int]
+ command: Optional[Command]
+ count_attempt: bool # either well-formed or typo
+ deletable: bool
+
+
+def _parse_command(line: str, bot_user: str) -> Optional[Command]:
+ if line.lower() == f"/u/{bot_user} reset".lower():
+ return Command.RESET
+ elif line.lower() in ["sidebar count", "current count"]:
+ return Command.REPORT
+ else:
+ return None
+
+
+def parse_update(payload_data: dict, curr_count: Optional[int], bot_user: str) -> ParsedUpdate:
+ # curr_count is the next number up, one more than the last count
+
+ NEW_LINE = object()
+ SPACE = object()
+
+ # flatten the update content to plain text
+ tree = BeautifulSoup(payload_data["body_html"], "html.parser")
+ worklist = tree.contents
+ out = [[]]
+ while worklist:
+ el = worklist.pop()
+ if isinstance(el, str):
+ out[-1].append(el)
+ elif el is SPACE:
+ out[-1].append(el)
+ elif el is NEW_LINE or el.name == "br" or el.name == "hr":
+ if out[-1]:
+ out.append([])
+ elif el.name in ["em", "strong", "del", "span", "sup", "code", "a", "th", "td"]:
+ worklist.extend(reversed(el.contents))
+ elif el.name in ["ul", "ol", "table", "thead", "tbody"]:
+ worklist.extend(reversed(el.contents))
+ elif el.name in ["li", "p", "div", "blockquote"] or re.match(r"h[1-6]$", el.name):
+ worklist.append(NEW_LINE)
+ worklist.extend(reversed(el.contents))
+ worklist.append(NEW_LINE)
+ elif el.name == "pre":
+ worklist.append(NEW_LINE)
+ worklist.extend([l] for l in reversed(el.text.splitlines()))
+ worklist.append(NEW_LINE)
+ elif el.name == "tr":
+ worklist.append(NEW_LINE)
+ for (i, cell) in enumerate(reversed(el.contents)):
+ worklist.append(cell)
+ if i != len(el.contents) - 1:
+ worklist.append(SPACE)
+ worklist.append(NEW_LINE)
+ else:
+ raise RuntimeError(f"can't parse tag {el.name}")
+
+ tmp_lines = (
+ "".join(" " if part is SPACE else part for part in parts).strip()
+ for parts in out
+ )
+ pre_strip_lines = list(filter(None, tmp_lines))
+
+ # normalize whitespace according to HTML rendering rules
+ # https://developer.mozilla.org/en-US/docs/Web/API/Document_Object_Model/Whitespace#explanation
+ stripped_lines = [
+ re.sub(" +", " ", l.replace("\t", " ").replace("\n", " ")).strip(" ")
+ for l in pre_strip_lines
+ ]
+
+ return _parse_from_lines(stripped_lines, curr_count, bot_user)
+
+
+def _parse_from_lines(lines: list[str], curr_count: Optional[int], bot_user: str) -> ParsedUpdate:
+ command = next(
+ filter(None, (_parse_command(l, bot_user) for l in lines)),
+ None
+ )
+ if lines:
+ # look for groups of digits (as many as possible) separated by a uniform separator from the valid set
+ first = lines[0]
+ match = re.match(
+ "(?P<v>v)?(?P<neg>-)?(?P<num>\\d+((?P<sep>[,. \u2009]|, )\\d+((?P=sep)\\d+)*)?)",
+ first,
+ re.ASCII, # only recognize ASCII digits
+ )
+ if match:
+ raw_digits = match["num"]
+ sep = match["sep"]
+ post = first[match.end() :]
+
+ zeros = False
+ while len(raw_digits) > 1 and raw_digits[0] == "0":
+ zeros = True
+ raw_digits = raw_digits.removeprefix("0").removeprefix(sep or "")
+
+ parts = raw_digits.split(sep) if sep else [raw_digits]
+ lone = len(lines) == 1 and (not post or post.isspace())
+ typo = False
+ if lone:
+ all_parts_valid = (
+ sep is None
+ or (
+ 1 <= len(parts[0]) <= 3
+ and all(len(p) == 3 for p in parts[1:])
+ )
+ )
+ if match["v"] and len(parts) == 1 and len(parts[0]) <= 2:
+ # failed paste of leading digits
+ typo = True
+ elif match["v"] and all_parts_valid:
+ # v followed by count
+ typo = True
+ elif curr_count is not None and abs(curr_count) >= 100 and bool(match["neg"]) == (curr_count < 0):
+ goal_parts = _separate(str(abs(curr_count)))
+ partials = [
+ goal_parts[: -1] + [goal_parts[-1][: -1]], # missing last digit
+ goal_parts[: -1] + [goal_parts[-1][: -2]], # missing last two digits
+ goal_parts[: -1] + [goal_parts[-1][: -2] + goal_parts[-1][-1]], # missing second-last digit
+ ]
+ if parts in partials:
+ # missing any of last two digits
+ typo = True
+ elif parts in [p[: -1] + [p[-1] + goal_parts[0]] + goal_parts[1 :] for p in partials]:
+ # double paste
+ typo = True
+
+ if match["v"] or zeros or typo or (parts == ["0"] and match["neg"]):
+ number = None
+ count_attempt = True
+ deletable = lone
+ else:
+ if curr_count is not None and sep and sep.isspace():
+ # Presume that the intended count consists of as many valid digit groups as necessary to match the
+ # number of digits in the expected count, if possible.
+ digit_count = len(str(abs(curr_count)))
+ use_parts = []
+ accum = 0
+ for (i, part) in enumerate(parts):
+ part_valid = len(part) <= 3 if i == 0 else len(part) == 3
+ if part_valid and accum < digit_count:
+ use_parts.append(part)
+ accum += len(part)
+ else:
+ break
+
+ # could still be a no-separator count with some extra digit groups on the same line
+ if not use_parts:
+ use_parts = [parts[0]]
+
+ lone = lone and len(use_parts) == len(parts)
+ else:
+ # current count is unknown or no separator was used
+ use_parts = parts
+
+ digits = "".join(use_parts)
+ number = -int(digits) if match["neg"] else int(digits)
+ special = (
+ curr_count is not None
+ and abs(number - curr_count) <= 25
+ and _is_special_number(number)
+ )
+ deletable = lone and not special
+ if len(use_parts) == len(parts) and post and not post[0].isspace():
+ count_attempt = curr_count is not None and abs(number - curr_count) <= 25
+ number = None
+ else:
+ count_attempt = True
+ else:
+ # no count attempt found
+ number = None
+ count_attempt = False
+ deletable = False
+ else:
+ # no lines in update
+ number = None
+ count_attempt = False
+ deletable = True
+
+ return ParsedUpdate(
+ number = number,
+ command = command,
+ count_attempt = count_attempt,
+ deletable = deletable,
+ )
+
+
+def _separate(digits: str) -> list[str]:
+ mod = len(digits) % 3
+ out = []
+ if mod:
+ out.append(digits[: mod])
+ out.extend(digits[i : i + 3] for i in range(mod, len(digits), 3))
+ return out
+
+
+def _is_special_number(num: int) -> bool:
+ num_str = str(num)
+ return bool(
+ num % 1000 in [0, 1, 333, 999]
+ or (num > 10_000_000 and "".join(reversed(num_str)) == num_str)
+ or re.match(r"(.+)\1+$", num_str) # repeated sequence
+ )
projects / counting.git / commitdiff