From: Andrea Cornell Date: Wed, 2 Nov 2022 01:05:49 +0000 (-0400) Subject: joinbot build/packaging X-Git-Tag: strikebot-0.0.7~10 X-Git-Url: https://jcornell.net/gitweb/gitweb.cgi?a=commitdiff_plain;h=d32d5877d22f748c1f5ef1ae1ca79bd879d6935a;p=counting.git joinbot build/packaging --- diff --git a/joinbot/.gitignore b/joinbot/.gitignore new file mode 100644 index 0000000..edd9d60 --- /dev/null +++ b/joinbot/.gitignore @@ -0,0 +1,2 @@ +build/ +dist/ diff --git a/joinbot/README.md b/joinbot/README.md new file mode 100644 index 0000000..3af2e1d --- /dev/null +++ b/joinbot/README.md @@ -0,0 +1,49 @@ +#Architecture + +background.py executed periodically (every 25 min). +service.py is a constantly-running service, uses postgresql asynchronous notifications to detect when there is work to do. +See file headers of the above to learn about their responsibilities. +background.py and service.py act on a "service"; this is sort of a multi-tenant capability. The default service name is "autojoin"; to use a different service name set the environment variable LIVEAUTOJOINSERVICE +service name is also used to find the database via "~/.pg\_service.conf" + +web.py is the Flask (wsgi) web application which handles /authorize, /invite, /ticket. + +flow: +1. user clicks join link (/authorize) in thread sidebar +2. app validates `thread` argument +3. app redirects to reddit OAuth2 /api/v1/authorize, state is : +4. user clicks 'Allow' +5. reddit redirects to /invite +6. app creates ticket linked to authorization code and `NOTIFY live_autojoin` +7. app redirects to /ticket (templates/status.html - auto-refreshing) +7. service.py retrieves authorization code to create authorization +8. service.py fills in ticket username using authorization +9. service.py invites user using admin authorization +10. service.py accepts invite using authorization +11. auto-refreshing /ticket starts redirecting back to the live event + +# Roadmap/wishlist +- allowed events: load more than 1 page of search results +- allowed events: search linkpost self text and "thread directory" wiki page for links, not just linkpost links +- when accepting admin invite, update reddit\_liveupdate\_event\_contributor table like when inviting users +- don't mention users in announcements +- respect sr bans +- check whether we will succeed (have permission, have scope, not too many outstanding invites) before adding ticket +- configurable (wiki page) allow/deny list for events +- invite sr moderators with extra permissions +- handle LIVEUPDATE\_TOO\_MANY\_INVITES (or whatever) +- actually report status/errors on /ticket +- handle no admin permission in `invite\_contributor` +- ticket processing rate limit (max 3 tickets in 60 seconds) - and if waiting on ratelimit, say so in status.html +- read modmail (and PMs?) and create tickets for messages with "inviteme" commands +- sync /live/mine (or allowed threads by\_id) to reddit\_liveupdate\_event table (background.py) +- include event title in error page's /message/compose link template +- after accepting admin invite, send PM to event creator (we don't know who created a thread, but we could find out who posted it in /r/livecounting and check if they have `settings` permission) with instructions for adding join link +- remove everyone's "close" permission? +- should be open-sourced, but needs deployment documentation and database schema script and pg\_reddit open-sourced first + +- find and fix DoS opportunities (something invalid submitted through web.py crashes service.py): better now. +- send "/u/username has joined this thread" updates: done. +- sync event contributor _invites_, not just contributors: done +- decide how to handle LIVEUPDATE\_ALREADY\_CONTRIBUTOR when `invite\_contributor`ing: done +- speculatively update invite table after `invite\_contributor`, speculatively update contributor table after `accept\_contributor\_invite`: done diff --git a/joinbot/deploy b/joinbot/deploy new file mode 100755 index 0000000..02fd157 --- /dev/null +++ b/joinbot/deploy @@ -0,0 +1,7 @@ +#!/bin/bash + +echo "Ok, but why aren't you using a package manager" + +#tar cz systemd | ssh root@bingus.internet6.net. tar xzC /etc/systemd/system +python3 -m build --no-isolation --wheel +tar cC dist joinbot-0.1.0-py3-none-any.whl | ssh anders@bingus.internet6.net. tar xC /var/tmp \&\& sudo pip3 install -U /var/tmp/joinbot-0.1.0-py3-none-any.whl --no-deps \&\& rm /var/tmp/joinbot-0.1.0-py3-none-any.whl diff --git a/joinbot/joinbot/announce.py b/joinbot/joinbot/announce.py new file mode 100644 index 0000000..0953b1c --- /dev/null +++ b/joinbot/joinbot/announce.py @@ -0,0 +1,59 @@ +# announce.py - announce new users + +import select +import urllib.parse +import urllib.request +import urllib.error +import base64 +import json + +from .common import connect_db, service_name + +POLL_INTERVAL=3600 + +def main(): + (cn, cr) = connect_db() + + cr.execute("LISTEN live_autojoin") + + while True: + cn.poll() + cn.notifies.clear() + work = False + while True: + cr.execute("BEGIN") + # - invited less than 1 minute ago + # - no announcement of same user in same event less than 30 days ago + # - user has update permission (not banned) + # - admin has update permission + # - admin authorization has submit scope + cr.execute("""SELECT access_token, admin_username, ticket.event_flake, username FROM live_autojoin_ticket AS ticket JOIN live_autojoin_service ON service_name=name JOIN live_autojoin_admin_authorization USING (service_name) JOIN reddit_app_authorization ON authorization_id=id JOIN reddit_liveupdate_event_contributor AS userrel ON userrel."type"='contributor' AND ticket.event_flake=userrel.event_flake AND username=userrel.name JOIN reddit_liveupdate_event_contributor AS adminrel ON adminrel."type"='contributor' AND ticket.event_flake = adminrel.event_flake AND admin_username = adminrel.name WHERE service_name = %s AND status = 'ok' AND updated_at + '1m' > CURRENT_TIMESTAMP AND NOT EXISTS (SELECT * FROM live_autojoin_announcement WHERE for_username = username AND event_flake = ticket.event_flake AND at + '30d' > updated_at) AND has_permission('update', userrel.permissions) AND has_permission('update', adminrel.permissions) AND array_position(scope, 'submit') IS NOT NULL LIMIT 1""", (service_name,)) + try: + [(access_token, admin_username, event_flake, username)] = cr.fetchall() + except ValueError: + break + else: + work = True + escaped_username = username.replace('_', '\\_') + usertext = f'*[\\/u\\/{ escaped_username }](/user/{ escaped_username }) has joined this thread*' + body = urllib.parse.urlencode({ 'api_type': 'json', 'body': usertext }).encode('utf-8') + req = urllib.request.Request(f'https://oauth.reddit.com/api/live/{event_flake}/update', data=body, method='POST') + req.add_header('Authorization', 'Bearer {}'.format(access_token)) + req.add_header('User-Agent', 'autojoin/0.1.0') + try: + res = json.load(urllib.request.urlopen(req)) + except urllib.error.HTTPError as e: + if e.code == 403: + admin_username = None + usertext = None + else: + raise + cr.execute("INSERT INTO live_autojoin_announcement (at, event_flake, for_username, author, body) VALUES (CURRENT_TIMESTAMP, %s, %s, %s, %s)", (event_flake, username, admin_username, usertext)) + finally: + cn.commit() + + cn.poll() + if work or len(cn.notifies) > 0: + continue + + select.select([cn], [], [], POLL_INTERVAL) diff --git a/joinbot/joinbot/background.py b/joinbot/joinbot/background.py new file mode 100644 index 0000000..abfe534 --- /dev/null +++ b/joinbot/joinbot/background.py @@ -0,0 +1,182 @@ +# background.py - jobs that are not time-critical. +# - refresh access tokens +# - sync PMs +# - sync allowed threads (subreddit links) +# - accept admin invites +# - sync contributor lists +# - TODO sync subreddit bans and moderators +# - TODO sync modmail (to receive joining requests to modmail) + +# roadmap +# - for PM, allowed threads, and modmail sync, create a lightweight version to run frequently to handle new events quickly--distinct from the more expensive "full sync" that is implemented here. + +import urllib.request +import urllib.parse + +import json +import base64 +import re + +from .common import connect_db, service_name + +def main(): + + # + # refresh tokens + # + + (cn, cr) = connect_db() + + def do_refresh_token(client_id, client_secret, refresh_token): + body = urllib.parse.urlencode({ 'grant_type': 'refresh_token', 'refresh_token': refresh_token }).encode('utf-8') + req = urllib.request.Request('https://www.reddit.com/api/v1/access_token', data=body, method='POST') + auth = base64.b64encode('{}:{}'.format(client_id, client_secret).encode('utf-8')).decode('utf-8') + req.add_header('Authorization', 'Basic {}'.format(auth)) + req.add_header('User-Agent', 'autojoin/0.1.0') + res = urllib.request.urlopen(req) + return json.load(res) + + while True: + cr.execute("BEGIN") + cr.execute("SELECT authorization_id, client_id, client_secret, refresh_token FROM live_autojoin_admin_authorization JOIN reddit_app_authorization ON authorization_id=id JOIN reddit_app USING (client_id) WHERE service_name = %s AND refresh_token IS NOT NULL AND expires < CURRENT_TIMESTAMP + '30min' LIMIT 1 FOR NO KEY UPDATE OF reddit_app_authorization", [service_name]) + try: + [(authorization_id, client_id, client_secret, refresh_token)] = cr.fetchall() + except ValueError: + cn.rollback() + break + else: + print('refreshing token for authorization {}'.format(authorization_id)) + new_token = do_refresh_token(client_id, client_secret, refresh_token) + cr.execute("UPDATE reddit_app_authorization SET access_token = %s, refresh_token = %s, scope = regexp_split_to_array(%s, ' ')::reddit_app_scope[], expires = CURRENT_TIMESTAMP + make_interval(secs => %s) WHERE id = %s", (new_token['access_token'], new_token['refresh_token'], new_token['scope'], new_token['expires_in'], authorization_id)) + cn.commit() + + # + # load PMs + # + + def flatten_privatemessage_listing(json_): + assert json_['kind'] == 'Listing' + for p in json_['data']['children']: + assert p['kind'] == 't4' + yield p['data'] + replies = p['data']['replies'] + if replies: + assert replies['kind'] == 'Listing' + for r in replies['data']['children']: + assert p['kind'] == 't4' + yield p['data'] + + def privatemessage_to_tuple(data): + id_ = data['id'] + parent_id = None + if data['parent_id'] is not None: + parent_id = data['parent_id'].split('_',maxsplit=1)[1] + assert int(parent_id, 36) == data['first_message'] + created = data['created_utc'] + sr = data['subreddit'] + author = None if data['author'] == '[deleted]' else data['author'] + if data['dest'].startswith('#'): + # modmail (destination is subreddit) + dest = None + dest_is_sr = True + else: + # destination is user + dest = data['dest'] + dest_is_sr = False + subject = data['subject'] + body = data['body'] + return (id_, parent_id, created, sr, author, dest, dest_is_sr, subject, body) + + cr.execute("BEGIN") + cr.execute("SELECT sr, access_token FROM live_autojoin_service JOIN live_autojoin_admin_authorization ON name = service_name JOIN reddit_app_authorization ON authorization_id=id JOIN reddit_app ON reddit_app.client_id=reddit_app_authorization.client_id WHERE service_name = %s", [service_name]) + [(sr, access_token)] = cr.fetchall() + cr.execute('CREATE TEMPORARY TABLE privatemessage_load (LIKE reddit_privatemessage INCLUDING INDEXES)') + after = None + while True: + if after: + rsc = 'https://oauth.reddit.com/message/messages?raw_json=1&limit=100&after={}'.format(after) + else: + rsc = 'https://oauth.reddit.com/message/messages?raw_json=1&limit=100' + req = urllib.request.Request(rsc, method='GET') + req.add_header('Authorization', 'Bearer {}'.format(access_token)) + req.add_header('User-Agent', 'autojoin/0.1.0') + res = json.load(urllib.request.urlopen(req)) + tuples = (privatemessage_to_tuple(d) for d in flatten_privatemessage_listing(res)) + cr.executemany('INSERT INTO privatemessage_load (id, parent_id, created, sr, author, dest, dest_is_sr, subject, body) VALUES (%s, %s, to_timestamp(%s), %s, %s, %s, %s, %s, %s)', tuples) + if 'after' in res: + after = res['after'] + else: + break + cr.execute("INSERT INTO reddit_privatemessage (id, parent_id, created, sr, author, dest, dest_is_sr, subject, body) SELECT id, parent_id, created, sr, author, dest, dest_is_sr, subject, body FROM privatemessage_load ON CONFLICT ON CONSTRAINT reddit_privatemessage_pkey DO NOTHING") + cr.execute("DROP TABLE privatemessage_load") + cn.commit() + + # + # build allowed thread list + # TODO look beyond first page + # TODO wiki config page for additional threads + # + + def flake_from_url(url_): + result = re.fullmatch('https?://[a-z]+\.reddit\.com/live/([a-z0-9]+)/?(?:\?.*)?', url_) + return result and result.group(1) + + def allowed_threads(): + req = urllib.request.Request('https://oauth.reddit.com/r/livecounting/search?q=url%3Alive+site%3Areddit.com+self%3Ano&restrict_sr=on&include_over_18=on&sort=new&t=all&limit=100', method='GET') + req.add_header('Authorization', 'Bearer {}'.format(access_token)) + req.add_header('User-Agent', 'autojoin/0.1.0') + res = json.load(urllib.request.urlopen(req)) + flakes = (flake_from_url(thing['data']['url']) for thing in res['data']['children'] if thing['data']['is_self'] is False) + return set((f for f in flakes if f)) + + cr.execute("BEGIN") + #cr.execute("DELETE FROM live_autojoin_allowed_event WHERE service_name = %s", (service_name,)) + cr.executemany("INSERT INTO live_autojoin_allowed_event (service_name, event_flake) VALUES (%s, %s) ON CONFLICT (service_name, event_flake) DO NOTHING", ((service_name, f) for f in allowed_threads())) + cn.commit() + + # accept admin invites + + cr.execute("BEGIN") + cr.execute("SELECT id, body FROM reddit_privatemessage JOIN live_autojoin_service ON dest = admin_username WHERE author = 'reddit' AND NOT dest_is_sr AND parent_id IS NULL AND subject LIKE 'invitation to contribute to %%' AND NOT EXISTS (SELECT * FROM live_autojoin_admin_invite WHERE privatemessage_id = id AND name = %s)", (service_name,)) + for (id_, body) in cr.fetchall(): + match = re.search('/live/([a-z0-9]+)/contributors', body) + event_flake = match and match.group(1) + result = None + if event_flake: + body = urllib.parse.urlencode({ 'api_type': 'json' }).encode('utf-8') + req = urllib.request.Request('https://oauth.reddit.com/api/live/{}/accept_contributor_invite'.format(event_flake), method='POST', data=body) + req.add_header('Authorization', 'Bearer {}'.format(access_token)) + req.add_header('User-Agent', 'autojoin/0.1.0') + urllib.request.urlopen(req) + result = 'ok' + cr.execute("INSERT INTO live_autojoin_admin_invite (privatemessage_id, event_flake, result) VALUES (%s, %s, %s)", [id_, event_flake, result]) + cn.commit() + + # load contributor lists + + cr.execute("BEGIN") + cr.execute("SELECT event_flake FROM live_autojoin_allowed_event WHERE service_name = %s", (service_name,)) + cn.commit() + for (event_flake,) in cr.fetchall(): + req = urllib.request.Request('https://oauth.reddit.com/live/{}/contributors'.format(event_flake), method='GET') + req.add_header('Authorization', 'Bearer {}'.format(access_token)) + req.add_header('User-Agent', 'autojoin/0.1.0') + contributors_lists = json.load(urllib.request.urlopen(req)) + # endpoint returns two listings, contributors and contributor _invites_, in JSON sequence. + # if not a contributor, cannot see contributor invites. + # in that case, no JSON sequence--simply the single listing is returned--annoying. + if isinstance(contributors_lists, dict): + empty_listing = {'kind': 'UserList', 'data': {'children': []}} + contributors_lists = [contributors_lists, empty_listing] + assert all((l['kind'] == 'UserList' for l in contributors_lists)) + contributors = ((event_flake, c['name'], '{{{}}}'.format(','.join(c['permissions'])), ty) for (ty, l) in zip(('contributor', 'invite'), contributors_lists) for c in l['data']['children']) + cr.execute("BEGIN") + cr.execute("DELETE FROM reddit_liveupdate_event_contributor WHERE event_flake = %s", (event_flake,)) + cr.executemany("INSERT INTO reddit_liveupdate_event_contributor (event_flake, name, permissions, \"type\") VALUES (%s, %s, %s::text[], %s)", contributors) + cn.commit() + print(event_flake) + + # TODO load subreddit bans (and moderators) + # TODO load modmail for joining requests + + cn.close() diff --git a/joinbot/joinbot/common.py b/joinbot/joinbot/common.py new file mode 100644 index 0000000..0e032d3 --- /dev/null +++ b/joinbot/joinbot/common.py @@ -0,0 +1,9 @@ +import psycopg2 +import os + +service_name = os.environ.get('LIVEAUTOJOINSERVICE', 'autojoin') + +def connect_db(): + cn = psycopg2.connect("service={}".format(service_name)) + cr = cn.cursor() + return (cn, cr) diff --git a/joinbot/joinbot/service.py b/joinbot/joinbot/service.py new file mode 100644 index 0000000..8246ead --- /dev/null +++ b/joinbot/joinbot/service.py @@ -0,0 +1,161 @@ +# service.py - invite processing + +#- exchange authentication codes +#- fetch authorization identity to populate ticket username +#- invite users +#- accept invites + +import select +import urllib.parse +import urllib.request +import urllib.error +import base64 +import json + +from .common import connect_db, service_name + +POLL_INTERVAL=3600 + +def main(): + (cn, cr) = connect_db() + + cr.execute("LISTEN live_autojoin") + + while True: + cn.poll() + cn.notifies.clear() + work = False + while True: + cr.execute("BEGIN") + cr.execute("SELECT client_id, client_secret, redirect_uri, id, code FROM live_autojoin_service JOIN reddit_app USING (client_id) JOIN live_autojoin_ticket ON name = service_name JOIN reddit_app_authorization_code ON id = authorization_code_id WHERE authorization_id IS NULL AND service_name = %s LIMIT 1 FOR UPDATE OF live_autojoin_ticket, reddit_app_authorization_code", (service_name,)) + try: + [(client_id, client_secret, redirect_uri, authorization_code_id, code)] = cr.fetchall() + except ValueError: + break + else: + work = True + body = urllib.parse.urlencode({ 'grant_type': 'authorization_code', 'code': code, 'redirect_uri': redirect_uri }).encode('utf-8') + req = urllib.request.Request('https://www.reddit.com/api/v1/access_token', data=body, method='POST') + auth = base64.b64encode('{}:{}'.format(client_id, client_secret).encode('utf-8')).decode('utf-8') + req.add_header('Authorization', 'Basic {}'.format(auth)) + req.add_header('User-Agent', 'autojoin/0.1.0') + try: + res = json.load(urllib.request.urlopen(req)) + except urllib.error.HTTPError as e: + if e.code == 404: + res = {'error': 'invalid_grant'} # supposing 404 really means this + else: + raise + if 'error' in res: + if res['error'] == 'invalid_grant': + cr.execute("DELETE FROM reddit_app_authorization_code WHERE id = %s", (authorization_code_id,)) + else: + raise ValueError("unrecognized error '{}'".format(res['error'])) + else: + assert res['token_type'] == 'bearer' + cr.execute("WITH q1 AS (INSERT INTO reddit_app_authorization (client_id, access_token, scope, expires) VALUES (%s, %s, regexp_split_to_array(%s, ' ')::reddit_app_scope[], CURRENT_TIMESTAMP + make_interval(secs => %s)) RETURNING id) UPDATE reddit_app_authorization_code SET authorization_id = q1.id FROM q1 WHERE reddit_app_authorization_code.id = %s", (client_id, res['access_token'], res['scope'], res['expires_in'], authorization_code_id)) + finally: + cn.commit() + + while True: + cr.execute("BEGIN") + cr.execute("SELECT reddit_app_authorization.id, access_token, ticket FROM live_autojoin_ticket JOIN reddit_app_authorization_code ON reddit_app_authorization_code.id=authorization_code_id JOIN reddit_app_authorization ON reddit_app_authorization.id = authorization_id WHERE service_name = %s AND username IS NULL AND array_position(scope, 'identity') IS NOT NULL LIMIT 1 FOR UPDATE OF live_autojoin_ticket", (service_name,)) + try: + [(authorization_id, access_token, ticket)] = cr.fetchall() + except ValueError: + break + else: + work = True + req = urllib.request.Request('https://oauth.reddit.com/api/v1/me', method='GET') + req.add_header('Authorization', 'Bearer {}'.format(access_token)) + req.add_header('User-Agent', 'autojoin/0.1.0') + try: + res = json.load(urllib.request.urlopen(req)) + except urllib.error.HTTPError as e: + if e.code == 401: + cr.execute("DELETE FROM reddit_app_authorization WHERE id = %s", (authorization_id,)) + else: + raise + else: + cr.execute("UPDATE live_autojoin_ticket SET username = %s WHERE service_name = %s AND ticket = %s", (res['name'], service_name, ticket)) + finally: + cn.commit() + + while True: + cr.execute("BEGIN") + cr.execute("SELECT access_token, ticket, event_flake, username FROM live_autojoin_ticket LEFT OUTER JOIN LATERAL (SELECT service_name, access_token FROM live_autojoin_admin_authorization JOIN reddit_app_authorization ON id=authorization_id WHERE service_name = live_autojoin_ticket.service_name LIMIT 1) q1 USING (service_name) WHERE service_name = %s AND username IS NOT NULL AND status IS NULL LIMIT 1 FOR NO KEY UPDATE OF live_autojoin_ticket", (service_name,)) + try: + [(access_token, ticket, event, username)] = cr.fetchall() + except ValueError: + break + else: + work = True + if access_token is None: + result = 'noadmin' + else: + body = urllib.parse.urlencode({ 'api_type': 'json', 'name': username, 'permissions': '+update', 'type': 'liveupdate_contributor_invite' }).encode('utf-8') + req = urllib.request.Request('https://oauth.reddit.com/api/live/{}/invite_contributor'.format(event), data=body, method='POST') + req.add_header('Authorization', 'Bearer {}'.format(access_token)) + req.add_header('User-Agent', 'autojoin/0.1.0') + try: + res = json.load(urllib.request.urlopen(req)) + except urllib.error.HTTPError as e: + if e.code == 403: + result = 'noadmin' + else: + raise + else: + errors = [er[0] for er in res['json']['errors']] + if not errors: + result = 'invited' + elif errors == ['LIVEUPDATE_ALREADY_CONTRIBUTOR']: + result = 'already_contributor_or_invited' + else: + raise RuntimeError(str(errors)) + if result == 'invited': + cr.execute("DELETE FROM reddit_liveupdate_event_contributor WHERE event_flake = %s AND name = %s", (event, username)) + cr.execute("INSERT INTO reddit_liveupdate_event_contributor (event_flake, name, permissions, type) VALUES (%s, %s, liveupdate_permission_set '+update', 'invite') ON CONFLICT DO NOTHING", (event, username)) + cr.execute("UPDATE live_autojoin_ticket SET status = %s, updated_at = CURRENT_TIMESTAMP WHERE service_name = %s AND ticket = %s", (result, service_name, ticket)) + finally: + cn.commit() + + while True: + cr.execute("BEGIN") + cr.execute("SELECT reddit_app_authorization.id, access_token, ticket, event_flake, username, status FROM live_autojoin_ticket JOIN reddit_app_authorization_code ON reddit_app_authorization_code.id=authorization_code_id JOIN reddit_app_authorization ON reddit_app_authorization.id = authorization_id WHERE service_name = %s AND status IN ('invited', 'already_contributor_or_invited') AND array_position(scope, 'livemanage') IS NOT NULL LIMIT 1 FOR NO KEY UPDATE OF live_autojoin_ticket", (service_name,)) + try: + [(authorization_id, access_token, ticket, event_flake, username, status)] = cr.fetchall() + except ValueError: + break + else: + work = True + body = urllib.parse.urlencode({ 'api_type': 'json' }).encode('utf-8') + req = urllib.request.Request('https://oauth.reddit.com/api/live/{}/accept_contributor_invite'.format(event_flake), data=body, method='POST') + req.add_header('Authorization', 'Bearer {}'.format(access_token)) + req.add_header('User-Agent', 'autojoin/0.1.0') + try: + res = json.load(urllib.request.urlopen(req)) + except urllib.error.HTTPError as e: + if e.code == 401: + cr.execute("DELETE FROM reddit_app_authorization WHERE id = %s", (authorization_id,)) + else: + raise + else: + errors = [er[0] for er in res['json']['errors']] + if not errors: + result = 'ok' + elif errors == ['LIVEUPDATE_NO_INVITE_FOUND']: + result = 'already_contributor' if status == 'already_contributor_or_invited' else None + else: + raise RuntimeError(str(errors)) + if result == 'ok': + cr.execute("UPDATE reddit_liveupdate_event_contributor SET type = 'contributor' WHERE event_flake = %s AND name = %s", (event, username)) + cr.execute("NOTIFY live_autojoin") # for announcements + cr.execute("UPDATE live_autojoin_ticket SET status = %s, updated_at = CURRENT_TIMESTAMP WHERE service_name = %s AND ticket = %s", (result, service_name, ticket)) + finally: + cn.commit() + + cn.poll() + if work or len(cn.notifies) > 0: + continue + + select.select([cn], [], [], POLL_INTERVAL) diff --git a/joinbot/joinbot/web/__init__.py b/joinbot/joinbot/web/__init__.py new file mode 100644 index 0000000..6d0f74f --- /dev/null +++ b/joinbot/joinbot/web/__init__.py @@ -0,0 +1,105 @@ +from flask import Flask, request, redirect, render_template, url_for + +import secrets +import re +import urllib.parse + +from ..common import connect_db + +DEFAULT_SERVICE = 'autojoin' + +app = Flask(__name__) + +def make_oauth_url(service_name, client_id, event, redirect_uri): + state = '{}:{}'.format(service_name, event) + scope = ' '.join(['identity','livemanage']) + qs = urllib.parse.urlencode({ + 'client_id': client_id, + 'response_type': 'code', + 'state': state, + 'redirect_uri': redirect_uri, + 'scope': scope, + 'duration': 'temporary' + }) + return 'https://www.reddit.com/api/v1/authorize?{}'.format(qs) + +def make_privatemessage_url(service_name, event): + if event and re.match('[a-z0-9]{10,}$', event): + body = '''I would like to join this thread: https://www.reddit.com/live/{1} + +(If you send this message with the following line intact, you will be invited automatically if possible) + +/autojoin service {0} event {1}'''.format(service_name, event) + elif event: + body = '''I would like to join this thread: {}'''.format(event) + else: + body = '''I would like to join this thread: (none)''' + assert re.match('[a-z0-9]+$', service_name) + qs = urllib.parse.urlencode({ + 'to': '/r/livecounting', + 'subject': 'Please invite me', + 'message': body + }) + return 'https://www.reddit.com/message/compose?{}'.format(qs) + +@app.route('/authorize') +def authorize(): + service_name = request.args.get('service', DEFAULT_SERVICE) + event = request.args.get('thread') + (cn, cr) = connect_db() + cr.execute("SELECT name, client_id, event_flake, redirect_uri FROM live_autojoin_allowed_event JOIN live_autojoin_service ON name = service_name JOIN reddit_app USING (client_id) WHERE service_name = %s AND event_flake = %s", (service_name, event)) + try: + [(service_name, client_id, event, redirect_uri)] = cr.fetchall() + except ValueError: + privatemessage_url = make_privatemessage_url(service_name, event) + return render_template("error.html", privatemessage_url=privatemessage_url) + else: + return redirect(make_oauth_url(service_name, client_id, event, redirect_uri), code=303) + finally: + cn.close() + +@app.route('/invite') +def invite(): + authorization_code = request.args.get('code') + state = request.args.get('state') + (service_name, event_flake) = state.split(':') + ticket = secrets.token_urlsafe() + (cn, cr) = connect_db() + cr.execute("BEGIN") + cr.execute("WITH q1 AS (INSERT INTO reddit_app_authorization_code (state, code) VALUES (%s, %s) RETURNING id) INSERT INTO live_autojoin_ticket (service_name, ticket, issued_at, event_flake, authorization_code_id, status) SELECT %s, %s, CURRENT_TIMESTAMP, %s, id, NULL FROM q1", (state, authorization_code, service_name, ticket, event_flake)) + cr.execute("NOTIFY live_autojoin") + cn.commit() + cn.close() + return redirect(url_for('ticket', service=service_name, ticket=ticket), code=303) + +@app.route('/ticket') +def ticket(): + service_name = request.args.get('service') + ticket = request.args.get('ticket') + (cn, cr) = connect_db() + cr.execute("SELECT event_flake, status FROM live_autojoin_ticket WHERE service_name = %s AND ticket = %s", (service_name, ticket)) + try: + [(event, status)] = cr.fetchall() + except ValueError: + event = None + status = 'error' + cn.close() + if status in ('ok', 'already_contributor'): + return redirect('https://www.reddit.com/live/{}'.format(event), code=308) + elif status == 'error': + privatemessage_url = make_privatemessage_url(service_name, event) + return render_template("error.html", privatemessage_url=privatemessage_url) + else: + privatemessage_url = make_privatemessage_url(service_name, event) + return render_template("status.html", privatemessage_url=privatemessage_url) + +# TODO +# - unallowed thread +# - allowed but disabled thread +# - authorization denied +# - indication of progress/errors + +#- exchange authentication codes +#- fetch authorization identity to populate ticket username +#- invite users +#- accept invites diff --git a/joinbot/joinbot/web/templates/error.html b/joinbot/joinbot/web/templates/error.html new file mode 100644 index 0000000..d393326 --- /dev/null +++ b/joinbot/joinbot/web/templates/error.html @@ -0,0 +1,10 @@ + + + + Autojoin: Error + + +

Error: {% block short %}unknown{% endblock %}

+ {% block long %}

Please message the moderators for assistance.

{% endblock %} + + diff --git a/joinbot/joinbot/web/templates/status.html b/joinbot/joinbot/web/templates/status.html new file mode 100644 index 0000000..dd87bea --- /dev/null +++ b/joinbot/joinbot/web/templates/status.html @@ -0,0 +1,13 @@ + + + + Autojoin: Pending + + + +

Inviting you to this thread

This should only take a few seconds. If many users are trying to join right now, you might have to wait longer.

You will be redirected back to the live thread once you have been added.

If this message persists, please message the moderators for assistance.

+ + diff --git a/joinbot/pyproject.toml b/joinbot/pyproject.toml new file mode 100644 index 0000000..8fe2f47 --- /dev/null +++ b/joinbot/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=42", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/joinbot/setup.cfg b/joinbot/setup.cfg new file mode 100644 index 0000000..f81d928 --- /dev/null +++ b/joinbot/setup.cfg @@ -0,0 +1,22 @@ +[metadata] +name = joinbot +version = 0.1.0 + +[options] +packages = + joinbot + joinbot.web +install_requires = + psycopg2 ~= 2.8 + flask ~= 2.2 +include_package_data = True + +[options.package_data] +joinbot.web = + templates/*.html + +[options.entry_points] +console_scripts = + joinbot-service = joinbot.service:main + joinbot-background = joinbot.background:main + joinbot-announce = joinbot.announce:main diff --git a/joinbot/setup.py b/joinbot/setup.py new file mode 100644 index 0000000..b908cbe --- /dev/null +++ b/joinbot/setup.py @@ -0,0 +1,3 @@ +import setuptools + +setuptools.setup() diff --git a/live-autojoin/README.md b/live-autojoin/README.md deleted file mode 100644 index 3af2e1d..0000000 --- a/live-autojoin/README.md +++ /dev/null @@ -1,49 +0,0 @@ -#Architecture - -background.py executed periodically (every 25 min). -service.py is a constantly-running service, uses postgresql asynchronous notifications to detect when there is work to do. -See file headers of the above to learn about their responsibilities. -background.py and service.py act on a "service"; this is sort of a multi-tenant capability. The default service name is "autojoin"; to use a different service name set the environment variable LIVEAUTOJOINSERVICE -service name is also used to find the database via "~/.pg\_service.conf" - -web.py is the Flask (wsgi) web application which handles /authorize, /invite, /ticket. - -flow: -1. user clicks join link (/authorize) in thread sidebar -2. app validates `thread` argument -3. app redirects to reddit OAuth2 /api/v1/authorize, state is : -4. user clicks 'Allow' -5. reddit redirects to /invite -6. app creates ticket linked to authorization code and `NOTIFY live_autojoin` -7. app redirects to /ticket (templates/status.html - auto-refreshing) -7. service.py retrieves authorization code to create authorization -8. service.py fills in ticket username using authorization -9. service.py invites user using admin authorization -10. service.py accepts invite using authorization -11. auto-refreshing /ticket starts redirecting back to the live event - -# Roadmap/wishlist -- allowed events: load more than 1 page of search results -- allowed events: search linkpost self text and "thread directory" wiki page for links, not just linkpost links -- when accepting admin invite, update reddit\_liveupdate\_event\_contributor table like when inviting users -- don't mention users in announcements -- respect sr bans -- check whether we will succeed (have permission, have scope, not too many outstanding invites) before adding ticket -- configurable (wiki page) allow/deny list for events -- invite sr moderators with extra permissions -- handle LIVEUPDATE\_TOO\_MANY\_INVITES (or whatever) -- actually report status/errors on /ticket -- handle no admin permission in `invite\_contributor` -- ticket processing rate limit (max 3 tickets in 60 seconds) - and if waiting on ratelimit, say so in status.html -- read modmail (and PMs?) and create tickets for messages with "inviteme" commands -- sync /live/mine (or allowed threads by\_id) to reddit\_liveupdate\_event table (background.py) -- include event title in error page's /message/compose link template -- after accepting admin invite, send PM to event creator (we don't know who created a thread, but we could find out who posted it in /r/livecounting and check if they have `settings` permission) with instructions for adding join link -- remove everyone's "close" permission? -- should be open-sourced, but needs deployment documentation and database schema script and pg\_reddit open-sourced first - -- find and fix DoS opportunities (something invalid submitted through web.py crashes service.py): better now. -- send "/u/username has joined this thread" updates: done. -- sync event contributor _invites_, not just contributors: done -- decide how to handle LIVEUPDATE\_ALREADY\_CONTRIBUTOR when `invite\_contributor`ing: done -- speculatively update invite table after `invite\_contributor`, speculatively update contributor table after `accept\_contributor\_invite`: done diff --git a/live-autojoin/announce.py b/live-autojoin/announce.py deleted file mode 100644 index 047ba77..0000000 --- a/live-autojoin/announce.py +++ /dev/null @@ -1,58 +0,0 @@ -# announce.py - announce new users - -import select -import urllib.parse -import urllib.request -import urllib.error -import base64 -import json - -from common import connect_db, service_name - -POLL_INTERVAL=3600 - -(cn, cr) = connect_db() - -cr.execute("LISTEN live_autojoin") - -while True: - cn.poll() - cn.notifies.clear() - work = False - while True: - cr.execute("BEGIN") - # - invited less than 1 minute ago - # - no announcement of same user in same event less than 30 days ago - # - user has update permission (not banned) - # - admin has update permission - # - admin authorization has submit scope - cr.execute("""SELECT access_token, admin_username, ticket.event_flake, username FROM live_autojoin_ticket AS ticket JOIN live_autojoin_service ON service_name=name JOIN live_autojoin_admin_authorization USING (service_name) JOIN reddit_app_authorization ON authorization_id=id JOIN reddit_liveupdate_event_contributor AS userrel ON userrel."type"='contributor' AND ticket.event_flake=userrel.event_flake AND username=userrel.name JOIN reddit_liveupdate_event_contributor AS adminrel ON adminrel."type"='contributor' AND ticket.event_flake = adminrel.event_flake AND admin_username = adminrel.name WHERE service_name = %s AND status = 'ok' AND updated_at + '1m' > CURRENT_TIMESTAMP AND NOT EXISTS (SELECT * FROM live_autojoin_announcement WHERE for_username = username AND event_flake = ticket.event_flake AND at + '30d' > updated_at) AND has_permission('update', userrel.permissions) AND has_permission('update', adminrel.permissions) AND array_position(scope, 'submit') IS NOT NULL LIMIT 1""", (service_name,)) - try: - [(access_token, admin_username, event_flake, username)] = cr.fetchall() - except ValueError: - break - else: - work = True - escaped_username = username.replace('_', '\\_') - usertext = f'*[\\/u\\/{ escaped_username }](/user/{ escaped_username }) has joined this thread*' - body = urllib.parse.urlencode({ 'api_type': 'json', 'body': usertext }).encode('utf-8') - req = urllib.request.Request(f'https://oauth.reddit.com/api/live/{event_flake}/update', data=body, method='POST') - req.add_header('Authorization', 'Bearer {}'.format(access_token)) - req.add_header('User-Agent', 'autojoin/0.1.0') - try: - res = json.load(urllib.request.urlopen(req)) - except urllib.error.HTTPError as e: - if e.code == 403: - admin_username = None - usertext = None - else: - raise - cr.execute("INSERT INTO live_autojoin_announcement (at, event_flake, for_username, author, body) VALUES (CURRENT_TIMESTAMP, %s, %s, %s, %s)", (event_flake, username, admin_username, usertext)) - finally: - cn.commit() - - cn.poll() - if work or len(cn.notifies) > 0: - continue - - select.select([cn], [], [], POLL_INTERVAL) diff --git a/live-autojoin/background.py b/live-autojoin/background.py deleted file mode 100644 index f5a9a87..0000000 --- a/live-autojoin/background.py +++ /dev/null @@ -1,180 +0,0 @@ -# background.py - jobs that are not time-critical. -# - refresh access tokens -# - sync PMs -# - sync allowed threads (subreddit links) -# - accept admin invites -# - sync contributor lists -# - TODO sync subreddit bans and moderators -# - TODO sync modmail (to receive joining requests to modmail) - -# roadmap -# - for PM, allowed threads, and modmail sync, create a lightweight version to run frequently to handle new events quickly--distinct from the more expensive "full sync" that is implemented here. - -import urllib.request -import urllib.parse - -import json -import base64 -import re - -from common import connect_db, service_name - -# -# refresh tokens -# - -(cn, cr) = connect_db() - -def do_refresh_token(client_id, client_secret, refresh_token): - body = urllib.parse.urlencode({ 'grant_type': 'refresh_token', 'refresh_token': refresh_token }).encode('utf-8') - req = urllib.request.Request('https://www.reddit.com/api/v1/access_token', data=body, method='POST') - auth = base64.b64encode('{}:{}'.format(client_id, client_secret).encode('utf-8')).decode('utf-8') - req.add_header('Authorization', 'Basic {}'.format(auth)) - req.add_header('User-Agent', 'autojoin/0.1.0') - res = urllib.request.urlopen(req) - return json.load(res) - -while True: - cr.execute("BEGIN") - cr.execute("SELECT authorization_id, client_id, client_secret, refresh_token FROM live_autojoin_admin_authorization JOIN reddit_app_authorization ON authorization_id=id JOIN reddit_app USING (client_id) WHERE service_name = %s AND refresh_token IS NOT NULL AND expires < CURRENT_TIMESTAMP + '30min' LIMIT 1 FOR NO KEY UPDATE OF reddit_app_authorization", [service_name]) - try: - [(authorization_id, client_id, client_secret, refresh_token)] = cr.fetchall() - except ValueError: - cn.rollback() - break - else: - print('refreshing token for authorization {}'.format(authorization_id)) - new_token = do_refresh_token(client_id, client_secret, refresh_token) - cr.execute("UPDATE reddit_app_authorization SET access_token = %s, refresh_token = %s, scope = regexp_split_to_array(%s, ' ')::reddit_app_scope[], expires = CURRENT_TIMESTAMP + make_interval(secs => %s) WHERE id = %s", (new_token['access_token'], new_token['refresh_token'], new_token['scope'], new_token['expires_in'], authorization_id)) - cn.commit() - -# -# load PMs -# - -def flatten_privatemessage_listing(json_): - assert json_['kind'] == 'Listing' - for p in json_['data']['children']: - assert p['kind'] == 't4' - yield p['data'] - replies = p['data']['replies'] - if replies: - assert replies['kind'] == 'Listing' - for r in replies['data']['children']: - assert p['kind'] == 't4' - yield p['data'] - -def privatemessage_to_tuple(data): - id_ = data['id'] - parent_id = None - if data['parent_id'] is not None: - parent_id = data['parent_id'].split('_',maxsplit=1)[1] - assert int(parent_id, 36) == data['first_message'] - created = data['created_utc'] - sr = data['subreddit'] - author = None if data['author'] == '[deleted]' else data['author'] - if data['dest'].startswith('#'): - # modmail (destination is subreddit) - dest = None - dest_is_sr = True - else: - # destination is user - dest = data['dest'] - dest_is_sr = False - subject = data['subject'] - body = data['body'] - return (id_, parent_id, created, sr, author, dest, dest_is_sr, subject, body) - -cr.execute("BEGIN") -cr.execute("SELECT sr, access_token FROM live_autojoin_service JOIN live_autojoin_admin_authorization ON name = service_name JOIN reddit_app_authorization ON authorization_id=id JOIN reddit_app ON reddit_app.client_id=reddit_app_authorization.client_id WHERE service_name = %s", [service_name]) -[(sr, access_token)] = cr.fetchall() -cr.execute('CREATE TEMPORARY TABLE privatemessage_load (LIKE reddit_privatemessage INCLUDING INDEXES)') -after = None -while True: - if after: - rsc = 'https://oauth.reddit.com/message/messages?raw_json=1&limit=100&after={}'.format(after) - else: - rsc = 'https://oauth.reddit.com/message/messages?raw_json=1&limit=100' - req = urllib.request.Request(rsc, method='GET') - req.add_header('Authorization', 'Bearer {}'.format(access_token)) - req.add_header('User-Agent', 'autojoin/0.1.0') - res = json.load(urllib.request.urlopen(req)) - tuples = (privatemessage_to_tuple(d) for d in flatten_privatemessage_listing(res)) - cr.executemany('INSERT INTO privatemessage_load (id, parent_id, created, sr, author, dest, dest_is_sr, subject, body) VALUES (%s, %s, to_timestamp(%s), %s, %s, %s, %s, %s, %s)', tuples) - if 'after' in res: - after = res['after'] - else: - break -cr.execute("INSERT INTO reddit_privatemessage (id, parent_id, created, sr, author, dest, dest_is_sr, subject, body) SELECT id, parent_id, created, sr, author, dest, dest_is_sr, subject, body FROM privatemessage_load ON CONFLICT ON CONSTRAINT reddit_privatemessage_pkey DO NOTHING") -cr.execute("DROP TABLE privatemessage_load") -cn.commit() - -# -# build allowed thread list -# TODO look beyond first page -# TODO wiki config page for additional threads -# - -def flake_from_url(url_): - result = re.fullmatch('https?://[a-z]+\.reddit\.com/live/([a-z0-9]+)/?(?:\?.*)?', url_) - return result and result.group(1) - -def allowed_threads(): - req = urllib.request.Request('https://oauth.reddit.com/r/livecounting/search?q=url%3Alive+site%3Areddit.com+self%3Ano&restrict_sr=on&include_over_18=on&sort=new&t=all&limit=100', method='GET') - req.add_header('Authorization', 'Bearer {}'.format(access_token)) - req.add_header('User-Agent', 'autojoin/0.1.0') - res = json.load(urllib.request.urlopen(req)) - flakes = (flake_from_url(thing['data']['url']) for thing in res['data']['children'] if thing['data']['is_self'] is False) - return set((f for f in flakes if f)) - -cr.execute("BEGIN") -#cr.execute("DELETE FROM live_autojoin_allowed_event WHERE service_name = %s", (service_name,)) -cr.executemany("INSERT INTO live_autojoin_allowed_event (service_name, event_flake) VALUES (%s, %s) ON CONFLICT (service_name, event_flake) DO NOTHING", ((service_name, f) for f in allowed_threads())) -cn.commit() - -# accept admin invites - -cr.execute("BEGIN") -cr.execute("SELECT id, body FROM reddit_privatemessage JOIN live_autojoin_service ON dest = admin_username WHERE author = 'reddit' AND NOT dest_is_sr AND parent_id IS NULL AND subject LIKE 'invitation to contribute to %%' AND NOT EXISTS (SELECT * FROM live_autojoin_admin_invite WHERE privatemessage_id = id AND name = %s)", (service_name,)) -for (id_, body) in cr.fetchall(): - match = re.search('/live/([a-z0-9]+)/contributors', body) - event_flake = match and match.group(1) - result = None - if event_flake: - body = urllib.parse.urlencode({ 'api_type': 'json' }).encode('utf-8') - req = urllib.request.Request('https://oauth.reddit.com/api/live/{}/accept_contributor_invite'.format(event_flake), method='POST', data=body) - req.add_header('Authorization', 'Bearer {}'.format(access_token)) - req.add_header('User-Agent', 'autojoin/0.1.0') - urllib.request.urlopen(req) - result = 'ok' - cr.execute("INSERT INTO live_autojoin_admin_invite (privatemessage_id, event_flake, result) VALUES (%s, %s, %s)", [id_, event_flake, result]) -cn.commit() - -# load contributor lists - -cr.execute("BEGIN") -cr.execute("SELECT event_flake FROM live_autojoin_allowed_event WHERE service_name = %s", (service_name,)) -cn.commit() -for (event_flake,) in cr.fetchall(): - req = urllib.request.Request('https://oauth.reddit.com/live/{}/contributors'.format(event_flake), method='GET') - req.add_header('Authorization', 'Bearer {}'.format(access_token)) - req.add_header('User-Agent', 'autojoin/0.1.0') - contributors_lists = json.load(urllib.request.urlopen(req)) - # endpoint returns two listings, contributors and contributor _invites_, in JSON sequence. - # if not a contributor, cannot see contributor invites. - # in that case, no JSON sequence--simply the single listing is returned--annoying. - if isinstance(contributors_lists, dict): - empty_listing = {'kind': 'UserList', 'data': {'children': []}} - contributors_lists = [contributors_lists, empty_listing] - assert all((l['kind'] == 'UserList' for l in contributors_lists)) - contributors = ((event_flake, c['name'], '{{{}}}'.format(','.join(c['permissions'])), ty) for (ty, l) in zip(('contributor', 'invite'), contributors_lists) for c in l['data']['children']) - cr.execute("BEGIN") - cr.execute("DELETE FROM reddit_liveupdate_event_contributor WHERE event_flake = %s", (event_flake,)) - cr.executemany("INSERT INTO reddit_liveupdate_event_contributor (event_flake, name, permissions, \"type\") VALUES (%s, %s, %s::text[], %s)", contributors) - cn.commit() - print(event_flake) - -# TODO load subreddit bans (and moderators) -# TODO load modmail for joining requests - -cn.close() diff --git a/live-autojoin/common.py b/live-autojoin/common.py deleted file mode 100644 index 0e032d3..0000000 --- a/live-autojoin/common.py +++ /dev/null @@ -1,9 +0,0 @@ -import psycopg2 -import os - -service_name = os.environ.get('LIVEAUTOJOINSERVICE', 'autojoin') - -def connect_db(): - cn = psycopg2.connect("service={}".format(service_name)) - cr = cn.cursor() - return (cn, cr) diff --git a/live-autojoin/deploy b/live-autojoin/deploy deleted file mode 100755 index eceeb7a..0000000 --- a/live-autojoin/deploy +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -echo "Ok, but why aren't you using a package manager" - -tar cz {cert,privkey}.pem {background,service,announce,common,web}.py templates systemd | ssh root@hatnd.acorn.pw tar xzC /opt/live-autojoin diff --git a/live-autojoin/service.py b/live-autojoin/service.py deleted file mode 100644 index a56c7b1..0000000 --- a/live-autojoin/service.py +++ /dev/null @@ -1,154 +0,0 @@ -# service.py - invite processing - -#- exchange authentication codes -#- fetch authorization identity to populate ticket username -#- invite users -#- accept invites - -import select -import urllib.parse -import urllib.request -import urllib.error -import base64 -import json - -from common import connect_db, service_name - -POLL_INTERVAL=3600 - -(cn, cr) = connect_db() - -cr.execute("LISTEN live_autojoin") - -while True: - cn.poll() - cn.notifies.clear() - work = False - while True: - cr.execute("BEGIN") - cr.execute("SELECT client_id, client_secret, redirect_uri, id, code FROM live_autojoin_service JOIN reddit_app USING (client_id) JOIN live_autojoin_ticket ON name = service_name JOIN reddit_app_authorization_code ON id = authorization_code_id WHERE authorization_id IS NULL AND service_name = %s LIMIT 1 FOR UPDATE OF live_autojoin_ticket, reddit_app_authorization_code", (service_name,)) - try: - [(client_id, client_secret, redirect_uri, authorization_code_id, code)] = cr.fetchall() - except ValueError: - break - else: - work = True - body = urllib.parse.urlencode({ 'grant_type': 'authorization_code', 'code': code, 'redirect_uri': redirect_uri }).encode('utf-8') - req = urllib.request.Request('https://www.reddit.com/api/v1/access_token', data=body, method='POST') - auth = base64.b64encode('{}:{}'.format(client_id, client_secret).encode('utf-8')).decode('utf-8') - req.add_header('Authorization', 'Basic {}'.format(auth)) - req.add_header('User-Agent', 'autojoin/0.1.0') - res = json.load(urllib.request.urlopen(req)) - if 'error' in res: - if res['error'] == 'invalid_grant': - cr.execute("DELETE FROM reddit_app_authorization_code WHERE id = %s", (authorization_code_id,)) - else: - raise ValueError("unrecognized error '{}'".format(res['error'])) - else: - assert res['token_type'] == 'bearer' - cr.execute("WITH q1 AS (INSERT INTO reddit_app_authorization (client_id, access_token, scope, expires) VALUES (%s, %s, regexp_split_to_array(%s, ' ')::reddit_app_scope[], CURRENT_TIMESTAMP + make_interval(secs => %s)) RETURNING id) UPDATE reddit_app_authorization_code SET authorization_id = q1.id FROM q1 WHERE reddit_app_authorization_code.id = %s", (client_id, res['access_token'], res['scope'], res['expires_in'], authorization_code_id)) - finally: - cn.commit() - - while True: - cr.execute("BEGIN") - cr.execute("SELECT reddit_app_authorization.id, access_token, ticket FROM live_autojoin_ticket JOIN reddit_app_authorization_code ON reddit_app_authorization_code.id=authorization_code_id JOIN reddit_app_authorization ON reddit_app_authorization.id = authorization_id WHERE service_name = %s AND username IS NULL AND array_position(scope, 'identity') IS NOT NULL LIMIT 1 FOR UPDATE OF live_autojoin_ticket", (service_name,)) - try: - [(authorization_id, access_token, ticket)] = cr.fetchall() - except ValueError: - break - else: - work = True - req = urllib.request.Request('https://oauth.reddit.com/api/v1/me', method='GET') - req.add_header('Authorization', 'Bearer {}'.format(access_token)) - req.add_header('User-Agent', 'autojoin/0.1.0') - try: - res = json.load(urllib.request.urlopen(req)) - except urllib.error.HTTPError as e: - if e.code == 401: - cr.execute("DELETE FROM reddit_app_authorization WHERE id = %s", (authorization_id,)) - else: - raise - else: - cr.execute("UPDATE live_autojoin_ticket SET username = %s WHERE service_name = %s AND ticket = %s", (res['name'], service_name, ticket)) - finally: - cn.commit() - - while True: - cr.execute("BEGIN") - cr.execute("SELECT access_token, ticket, event_flake, username FROM live_autojoin_ticket LEFT OUTER JOIN LATERAL (SELECT service_name, access_token FROM live_autojoin_admin_authorization JOIN reddit_app_authorization ON id=authorization_id WHERE service_name = live_autojoin_ticket.service_name LIMIT 1) q1 USING (service_name) WHERE service_name = %s AND username IS NOT NULL AND status IS NULL LIMIT 1 FOR NO KEY UPDATE OF live_autojoin_ticket", (service_name,)) - try: - [(access_token, ticket, event, username)] = cr.fetchall() - except ValueError: - break - else: - work = True - if access_token is None: - result = 'noadmin' - else: - body = urllib.parse.urlencode({ 'api_type': 'json', 'name': username, 'permissions': '+update', 'type': 'liveupdate_contributor_invite' }).encode('utf-8') - req = urllib.request.Request('https://oauth.reddit.com/api/live/{}/invite_contributor'.format(event), data=body, method='POST') - req.add_header('Authorization', 'Bearer {}'.format(access_token)) - req.add_header('User-Agent', 'autojoin/0.1.0') - try: - res = json.load(urllib.request.urlopen(req)) - except urllib.error.HTTPError as e: - if e.code == 403: - result = 'noadmin' - else: - raise - else: - errors = [er[0] for er in res['json']['errors']] - if not errors: - result = 'invited' - elif errors == ['LIVEUPDATE_ALREADY_CONTRIBUTOR']: - result = 'already_contributor_or_invited' - else: - raise RuntimeError(str(errors)) - if result == 'invited': - cr.execute("DELETE FROM reddit_liveupdate_event_contributor WHERE event_flake = %s AND name = %s", (event, username)) - cr.execute("INSERT INTO reddit_liveupdate_event_contributor (event_flake, name, permissions, type) VALUES (%s, %s, liveupdate_permission_set '+update', 'invite') ON CONFLICT DO NOTHING", (event, username)) - cr.execute("UPDATE live_autojoin_ticket SET status = %s, updated_at = CURRENT_TIMESTAMP WHERE service_name = %s AND ticket = %s", (result, service_name, ticket)) - finally: - cn.commit() - - while True: - cr.execute("BEGIN") - cr.execute("SELECT reddit_app_authorization.id, access_token, ticket, event_flake, username, status FROM live_autojoin_ticket JOIN reddit_app_authorization_code ON reddit_app_authorization_code.id=authorization_code_id JOIN reddit_app_authorization ON reddit_app_authorization.id = authorization_id WHERE service_name = %s AND status IN ('invited', 'already_contributor_or_invited') AND array_position(scope, 'livemanage') IS NOT NULL LIMIT 1 FOR NO KEY UPDATE OF live_autojoin_ticket", (service_name,)) - try: - [(authorization_id, access_token, ticket, event_flake, username, status)] = cr.fetchall() - except ValueError: - break - else: - work = True - body = urllib.parse.urlencode({ 'api_type': 'json' }).encode('utf-8') - req = urllib.request.Request('https://oauth.reddit.com/api/live/{}/accept_contributor_invite'.format(event_flake), data=body, method='POST') - req.add_header('Authorization', 'Bearer {}'.format(access_token)) - req.add_header('User-Agent', 'autojoin/0.1.0') - try: - res = json.load(urllib.request.urlopen(req)) - except urllib.error.HTTPError as e: - if e.code == 401: - cr.execute("DELETE FROM reddit_app_authorization WHERE id = %s", (authorization_id,)) - else: - raise - else: - errors = [er[0] for er in res['json']['errors']] - if not errors: - result = 'ok' - elif errors == ['LIVEUPDATE_NO_INVITE_FOUND']: - result = 'already_contributor' if status == 'already_contributor_or_invited' else None - else: - raise RuntimeError(str(errors)) - if result == 'ok': - cr.execute("UPDATE reddit_liveupdate_event_contributor SET type = 'contributor' WHERE event_flake = %s AND name = %s", (event, username)) - cr.execute("NOTIFY live_autojoin") # for announcements - cr.execute("UPDATE live_autojoin_ticket SET status = %s, updated_at = CURRENT_TIMESTAMP WHERE service_name = %s AND ticket = %s", (result, service_name, ticket)) - finally: - cn.commit() - - cn.poll() - if work or len(cn.notifies) > 0: - continue - - select.select([cn], [], [], POLL_INTERVAL) diff --git a/live-autojoin/systemd/live-autojoin-announce.service b/live-autojoin/systemd/live-autojoin-announce.service deleted file mode 100644 index e00569c..0000000 --- a/live-autojoin/systemd/live-autojoin-announce.service +++ /dev/null @@ -1,8 +0,0 @@ -[Service] -WorkingDirectory=/opt/live-autojoin -User=counting -Group=counting -Environment=LIVEAUTOJOINSERVICE=autojoin -Type=simple -ExecStart=python3 announce.py -Restart=always diff --git a/live-autojoin/systemd/live-autojoin-cron.service b/live-autojoin/systemd/live-autojoin-cron.service deleted file mode 100644 index 929dbe3..0000000 --- a/live-autojoin/systemd/live-autojoin-cron.service +++ /dev/null @@ -1,7 +0,0 @@ -[Service] -WorkingDirectory=/opt/live-autojoin -User=counting -Group=counting -Environment=LIVEAUTOJOINSERVICE=autojoin -Type=oneshot -ExecStart=python3 background.py diff --git a/live-autojoin/systemd/live-autojoin-cron.timer b/live-autojoin/systemd/live-autojoin-cron.timer deleted file mode 100644 index 5843509..0000000 --- a/live-autojoin/systemd/live-autojoin-cron.timer +++ /dev/null @@ -1,3 +0,0 @@ -[Timer] -OnActiveSec=0 -OnUnitActiveSec=25min diff --git a/live-autojoin/systemd/live-autojoin-service.service b/live-autojoin/systemd/live-autojoin-service.service deleted file mode 100644 index d69c607..0000000 --- a/live-autojoin/systemd/live-autojoin-service.service +++ /dev/null @@ -1,8 +0,0 @@ -[Service] -WorkingDirectory=/opt/live-autojoin -User=counting -Group=counting -Environment=LIVEAUTOJOINSERVICE=autojoin -Type=simple -ExecStart=python3 service.py -Restart=always diff --git a/live-autojoin/systemd/live-autojoin-web.service b/live-autojoin/systemd/live-autojoin-web.service deleted file mode 100644 index 7b1ebba..0000000 --- a/live-autojoin/systemd/live-autojoin-web.service +++ /dev/null @@ -1,7 +0,0 @@ -[Service] -WorkingDirectory=/opt/live-autojoin -User=counting -Group=counting -Environment=SCRIPT_NAME=/autojoin LIVEAUTOJOINSERVICE=autojoin -Type=simple # `notify` if gunicorn >= 20 -ExecStart=/usr/bin/gunicorn3 web:app diff --git a/live-autojoin/systemd/live-autojoin-web.socket b/live-autojoin/systemd/live-autojoin-web.socket deleted file mode 100644 index be9f1a0..0000000 --- a/live-autojoin/systemd/live-autojoin-web.socket +++ /dev/null @@ -1,4 +0,0 @@ -[Socket] -ListenStream=/tmp/live-autojoin.socket -SocketUser=www-data -SocketGroup=www-data diff --git a/live-autojoin/systemd/live-autojoin.target b/live-autojoin/systemd/live-autojoin.target deleted file mode 100644 index a028573..0000000 --- a/live-autojoin/systemd/live-autojoin.target +++ /dev/null @@ -1,5 +0,0 @@ -[Unit] -Requires=live-autojoin-cron.timer live-autojoin-service.service live-autojoin-web.socket live-autojoin-announce.service - -[Install] -WantedBy=multi-user.target diff --git a/live-autojoin/templates/error.html b/live-autojoin/templates/error.html deleted file mode 100644 index d393326..0000000 --- a/live-autojoin/templates/error.html +++ /dev/null @@ -1,10 +0,0 @@ - - - - Autojoin: Error - - -

Error: {% block short %}unknown{% endblock %}

- {% block long %}

Please message the moderators for assistance.

{% endblock %} - - diff --git a/live-autojoin/templates/status.html b/live-autojoin/templates/status.html deleted file mode 100644 index dd87bea..0000000 --- a/live-autojoin/templates/status.html +++ /dev/null @@ -1,13 +0,0 @@ - - - - Autojoin: Pending - - - -

Inviting you to this thread

This should only take a few seconds. If many users are trying to join right now, you might have to wait longer.

You will be redirected back to the live thread once you have been added.

If this message persists, please message the moderators for assistance.

- - diff --git a/live-autojoin/web.py b/live-autojoin/web.py deleted file mode 100644 index 819cf97..0000000 --- a/live-autojoin/web.py +++ /dev/null @@ -1,105 +0,0 @@ -from flask import Flask, request, redirect, render_template, url_for - -import secrets -import re -import urllib.parse - -from common import connect_db - -DEFAULT_SERVICE = 'autojoin' - -app = Flask(__name__) - -def make_oauth_url(service_name, client_id, event, redirect_uri): - state = '{}:{}'.format(service_name, event) - scope = ' '.join(['identity','livemanage']) - qs = urllib.parse.urlencode({ - 'client_id': client_id, - 'response_type': 'code', - 'state': state, - 'redirect_uri': redirect_uri, - 'scope': scope, - 'duration': 'temporary' - }) - return 'https://www.reddit.com/api/v1/authorize?{}'.format(qs) - -def make_privatemessage_url(service_name, event): - if event and re.match('[a-z0-9]{10,}$', event): - body = '''I would like to join this thread: https://www.reddit.com/live/{1} - -(If you send this message with the following line intact, you will be invited automatically if possible) - -/autojoin service {0} event {1}'''.format(service_name, event) - elif event: - body = '''I would like to join this thread: {}'''.format(event) - else: - body = '''I would like to join this thread: (none)''' - assert re.match('[a-z0-9]+$', service_name) - qs = urllib.parse.urlencode({ - 'to': '/r/livecounting', - 'subject': 'Please invite me', - 'message': body - }) - return 'https://www.reddit.com/message/compose?{}'.format(qs) - -@app.route('/authorize') -def authorize(): - service_name = request.args.get('service', DEFAULT_SERVICE) - event = request.args.get('thread') - (cn, cr) = connect_db() - cr.execute("SELECT name, client_id, event_flake, redirect_uri FROM live_autojoin_allowed_event JOIN live_autojoin_service ON name = service_name JOIN reddit_app USING (client_id) WHERE service_name = %s AND event_flake = %s", (service_name, event)) - try: - [(service_name, client_id, event, redirect_uri)] = cr.fetchall() - except ValueError: - privatemessage_url = make_privatemessage_url(service_name, event) - return render_template("error.html", privatemessage_url=privatemessage_url) - else: - return redirect(make_oauth_url(service_name, client_id, event, redirect_uri), code=303) - finally: - cn.close() - -@app.route('/invite') -def invite(): - authorization_code = request.args.get('code') - state = request.args.get('state') - (service_name, event_flake) = state.split(':') - ticket = secrets.token_urlsafe() - (cn, cr) = connect_db() - cr.execute("BEGIN") - cr.execute("WITH q1 AS (INSERT INTO reddit_app_authorization_code (state, code) VALUES (%s, %s) RETURNING id) INSERT INTO live_autojoin_ticket (service_name, ticket, issued_at, event_flake, authorization_code_id, status) SELECT %s, %s, CURRENT_TIMESTAMP, %s, id, NULL FROM q1", (state, authorization_code, service_name, ticket, event_flake)) - cr.execute("NOTIFY live_autojoin") - cn.commit() - cn.close() - return redirect(url_for('ticket', service=service_name, ticket=ticket), code=303) - -@app.route('/ticket') -def ticket(): - service_name = request.args.get('service') - ticket = request.args.get('ticket') - (cn, cr) = connect_db() - cr.execute("SELECT event_flake, status FROM live_autojoin_ticket WHERE service_name = %s AND ticket = %s", (service_name, ticket)) - try: - [(event, status)] = cr.fetchall() - except ValueError: - event = None - status = 'error' - cn.close() - if status in ('ok', 'already_contributor'): - return redirect('https://www.reddit.com/live/{}'.format(event), code=308) - elif status == 'error': - privatemessage_url = make_privatemessage_url(service_name, event) - return render_template("error.html", privatemessage_url=privatemessage_url) - else: - privatemessage_url = make_privatemessage_url(service_name, event) - return render_template("status.html", privatemessage_url=privatemessage_url) - -# TODO -# - unallowed thread -# - allowed but disabled thread -# - authorization denied -# - indication of progress/errors - -#- exchange authentication codes -#- fetch authorization identity to populate ticket username -#- invite users -#- accept invites