Understand (replyto ...) style subjects. This is a bit complicated because it requires keeping track of what twts have been seen, and potentially updating a message if the target appears after the referer is fetched. Track this using an sqlite db. Also, use proper maildir filenames. We need to rewrite mails in some cases, and you're not supposed to edit a mail file after it's written to a maildir, so might as well switch to using proper names. diff /home/falsifian/co/jenny commit - 73a5ea812160c02ca13dd3856b26e314c9a08475 path + /home/falsifian/co/jenny blob - 23e350c83bae3bf25b6c49c47edba955204c6678 file + jenny --- jenny +++ jenny @@ -9,12 +9,14 @@ from enum import Enum, auto from getpass import getuser from hashlib import blake2b from json import dumps, loads -from os import getenv, listdir, makedirs, mkdir, rename, rmdir +from os import getenv, listdir, makedirs, mkdir, rename, rmdir, unlink from os.path import expanduser, getmtime, isfile, join +from random import randbytes from re import compile as re_compile, match from shlex import quote from ssl import CERT_NONE, PROTOCOL_TLS_CLIENT, SSLContext, TLSVersion -from socket import socket +from socket import gethostname, socket +import sqlite3 from subprocess import run from sys import stderr, stdin from tempfile import TemporaryDirectory @@ -28,6 +30,8 @@ from requests import get RE_HASH_MENTION = re_compile(r' *\(#([a-z0-9]{7})\) *(@<[^>]+>)') RE_MENTION_HASH = re_compile(r' *(@<[^>]+>) *\(#([a-z0-9]{7})\)') RE_HASH = re_compile(r' *\(#([a-z0-9]{7})\)') +# TODO: be more strict about finding URLs and timestamps in replyto subjects. +RE_REPLYTO = re_compile(r' *\(replyto ([^ ]+) ([^ ]+)\)') RE_NICK_INVALID = re_compile(r'[^a-zA-Z0-9]') @@ -274,8 +278,12 @@ def get_metadata_fields(s, key): return [i for i in out if i != ''] -def get_feed_states_dir(): - return join(getenv('XDG_CACHE_HOME', expanduser('~/.cache')), 'jenny') +def get_feed_states_dir(app): + if 'feed_states_dir' in app['config']: + # For overriding in tests + return app['config']['feed_states_dir'] + else: + return join(getenv('XDG_CACHE_HOME', expanduser('~/.cache')), 'jenny') def get_prev_archived_feed(content, feed_url): @@ -310,9 +318,9 @@ def get_prev_archived_feed(content, feed_url): return aurl -def get_stamp_file(name): +def get_stamp_file(app, name): try: - with open(join(get_feed_states_dir(), name), 'r', encoding='UTF-8') as fp: + with open(join(get_feed_states_dir(app), name), 'r', encoding='UTF-8') as fp: return loads(fp.read()) except FileNotFoundError: return {} @@ -326,7 +334,18 @@ def initialize_maildir(app): ): makedirs(i, exist_ok=True) +def initialize_sqlite(app): + makedirs(get_feed_states_dir(app), exist_ok=True) + app['sqlite_con'] = sqlite3.connect( + f'{get_feed_states_dir(app)}/db.sqlite') + app['sqlite_con'].isolation_level = None + app['sqlite_con'].execute('create table if not exists have_twt(hash, source_url, timestamp)') + app['sqlite_con'].execute('create table if not exists want_twt(url, timestamp, referer_filename)') +def close_sqlite(app): + app['sqlite_con'].close() + del app['sqlite_con'] + def find_thread_context(mail): blocks = mail.split('\n\n') assert len(blocks) >= 2 @@ -350,17 +369,6 @@ def find_thread_context(mail): return thread_hash, nick, url -def mail_file_exists(app, twt_hash): - if app['existing_twt_cache'] is None: - app['existing_twt_cache'] = set() - for d in ('cur', 'new'): - files = listdir(join(app['config']['maildir_target'], d)) - files = [i[:i.find(':')] for i in files] - app['existing_twt_cache'] |= set(files) - - return twt_hash in app['existing_twt_cache'] - - def make_twt_hash(url, twt_date, msg): created = twt_date.replace(microsecond=0).isoformat().replace('+00:00', 'Z') payload = f'{url}\n{created}\n{msg}' @@ -471,45 +479,116 @@ def process_feed( list_of_hashes = [] for line in twt_lines_from_content(content): - res = twt_line_to_mail( - app, - line, - configured_url, - url_for_hash, - nick_desc, - nick_address, - ) - if res is None: - continue - mail_body, twt_hash, twt_date = res - # Only used in fetch_context(). - if only_twt_hash and twt_hash != only_twt_hash: - continue + if only_twt_hash is not None: + # TODO: Avoid parsing and hashing the twt twice. + res = parse_raw_twt(content) + if res is None: + continue + if only_twt_hash != make_twt_hash(url_for_hash, res[0], res[1]): + continue + res = deliver_twt(app, line, configured_url, url_for_hash, nick_desc, nick_address) + if res is None: + continue + twt_hash, twt_date = res list_of_hashes.append(twt_hash) - - twt_stamp = twt_date.timestamp() - if lasttwt is not None and lasttwt >= twt_stamp: - continue if new_lasttwt is None or twt_stamp > new_lasttwt: new_lasttwt = twt_stamp new_lasthash = twt_hash - mailname_new = join(app['config']['maildir_target'], 'new', twt_hash) - mailname_tmp = join(app['config']['maildir_target'], 'tmp', twt_hash) - - if mail_file_exists(app, twt_hash): - continue - - with open(mailname_tmp, 'w', encoding='UTF-8') as fp: - fp.write(mail_body) - - rename(mailname_tmp, mailname_new) - return new_lasttwt, new_lasthash, list_of_hashes +def gen_maildir_filename(): + return(f'{int(time())}.R{randbytes(24).hex()}.{gethostname()}') +def deliver_twt(app, content, configured_url, canonical_url, nick_desc, nick_address, filename=None): + if filename is None: + filename = gen_maildir_filename() + + res = twt_line_to_mail( + app, + content, + configured_url, + canonical_url, + nick_desc, + nick_address, + ) + if res is None: + return None + mail_body, twt_hash, twt_date, want_twt = res + + twt_stamp = twt_date.timestamp() + + mailname_new = join(app['config']['maildir_target'], 'new', filename) + mailname_tmp = join(app['config']['maildir_target'], 'tmp', filename) + + already_have = app['sqlite_con'].execute( + 'select 0 from have_twt where hash = ? limit 1', + (twt_hash,) + ).fetchall() + if len(already_have) > 0: + return None + + with open(mailname_tmp, 'w', encoding='UTF-8') as fp: + fp.write(mail_body) + + rename(mailname_tmp, mailname_new) + raw_time = content.split('\t', 1)[0] + app['sqlite_con'].execute('insert into have_twt values(:hash, :url, :time)', + {'hash': twt_hash, 'url': canonical_url, 'time': raw_time}) + + if want_twt is not None: + app['sqlite_con'].execute( + 'insert into want_twt values(:url, :time, :filename)', + {'url': want_twt[0], 'time': want_twt[1], 'filename': filename}) + + need_rewrite = app['sqlite_con'].execute( + 'select referer_filename from want_twt where url = :url and timestamp = :time', + (canonical_url, raw_time) + ).fetchall() + for row in need_rewrite: + rewrite_twt(app, row[0], twt_hash) + +def find_mail_file(app, filename): + for d in ('new', 'cur'): + dir = join(app['config']['maildir_target'], d) + for member in listdir(dir): + base = member.split(':', maxsplit=2)[0] + if base == filename: + return join(dir, member) + # Maybe we should make this a custom error. + raise FileNotFoundError(filename) + +def rewrite_twt(app, filename, target_hash): + path = find_mail_file(app, filename) + old_file = open(path) + new_filename = gen_maildir_filename() + # TODO: deduplicate this code with similar in deliver_twt + mailname_new = join(app['config']['maildir_target'], 'new', new_filename) + mailname_tmp = join(app['config']['maildir_target'], 'tmp', new_filename) + new_file = open(mailname_tmp, 'w') + + # Find end of headers + for line in old_file: + # TODO: The line should be '\r\n' but we're writing mails without '\r'. + if line == '\n': + break + new_file.write(line) + # Insert new In-Reply-To header. + new_file.write(f'In-Reply-To: <{target_hash}@twtxt>\n\n') + # Copy the rest + new_file.write(old_file.read()) + new_file.close() + old_file.close() + + rename(mailname_tmp, mailname_new) + + app['sqlite_con'].execute( + 'delete from want_twt where referer_filename = (:filename)', + {'filename': filename}) + unlink(path) + def refresh_and_publish(app): # This is a shortcut: After posting a new twt or editing the file # locally, we process the local file to create new entries in the @@ -526,8 +605,8 @@ def refresh_self_locally(app): with open(app['config']['self_local'], 'r', encoding='UTF-8') as fp: content = fp.read() - lasttwts = get_stamp_file('lasttwts') - lasthash = get_stamp_file('lasthash') + lasttwts = get_stamp_file(app, 'lasttwts') + lasthash = get_stamp_file(app, 'lasthash') this_lasttwt, this_lasthash, _ = process_feed( app, @@ -537,15 +616,15 @@ def refresh_self_locally(app): lasttwts.get(app['config']['self_url']), ) - makedirs(expanduser(get_feed_states_dir()), exist_ok=True) + makedirs(expanduser(get_feed_states_dir(app)), exist_ok=True) if this_lasttwt is not None: lasttwts[app['config']['self_url']] = this_lasttwt - store_stamp_file('lasttwts', lasttwts) + store_stamp_file(app, 'lasttwts', lasttwts) if this_lasthash is not None: lasthash[app['config']['self_url']] = this_lasthash - store_stamp_file('lasthash', lasthash) + store_stamp_file(app, 'lasthash', lasthash) def read_config(): @@ -650,11 +729,11 @@ def read_config(): def retrieve_all(app): ret = True - lastmods = get_stamp_file('lastmods') - lastseen = get_stamp_file('lastseen') - lasttwts = get_stamp_file('lasttwts') - lasthash = get_stamp_file('lasthash') - lasthash_old = get_stamp_file('lasthash') + lastmods = get_stamp_file(app, 'lastmods') + lastseen = get_stamp_file(app, 'lastseen') + lasttwts = get_stamp_file(app, 'lasttwts') + lasthash = get_stamp_file(app, 'lasthash') + lasthash_old = get_stamp_file(app, 'lasthash') # Try to never fetch the same URL twice. This is especially # important when following "prev" fields, as those are set by the @@ -771,12 +850,12 @@ def retrieve_all(app): else: ret = False - makedirs(expanduser(get_feed_states_dir()), exist_ok=True) + makedirs(expanduser(get_feed_states_dir(app)), exist_ok=True) - store_stamp_file('lastmods', lastmods) - store_stamp_file('lastseen', lastseen) - store_stamp_file('lasttwts', lasttwts) - store_stamp_file('lasthash', lasthash) + store_stamp_file(app, 'lastmods', lastmods) + store_stamp_file(app, 'lastseen', lastseen) + store_stamp_file(app, 'lasttwts', lasttwts) + store_stamp_file(app, 'lasthash', lasthash) return ret @@ -988,6 +1067,14 @@ def search_for_hash(msg): return None +def search_for_replyto(msg): + m = RE_REPLYTO.match(msg) + if m: + return m.group(1), m.group(2) + else: + return None + + def seconds_to_human_approx(seconds): # Ranges: # @@ -1045,8 +1132,8 @@ def show_lastseen(app, last_file, not_found_string): print(to_print.strip()) -def store_stamp_file(name, dict_content): - with open(join(get_feed_states_dir(), name), 'w', encoding='UTF-8') as fp: +def store_stamp_file(app, name, dict_content): + with open(join(get_feed_states_dir(app, ), name), 'w', encoding='UTF-8') as fp: fp.write(dumps(dict_content, sort_keys=True, indent=4)) @@ -1058,9 +1145,22 @@ def twt_line_to_mail(app, line, url, url_for_hash, nic twt_hash = make_twt_hash(url_for_hash, twt_date, msg) in_reply_to = None + want_twt = None res = search_for_hash(msg) if res is not None: in_reply_to, _ = res + else: + # Look for a (replyto ...) style subject. + res = search_for_replyto(msg) + if res is not None: + rows = app['sqlite_con'].execute( + 'select hash from have_twt where source_url = :url and timestamp = :time', + {'url': res[0], 'time': res[1]} + ).fetchall() + if len(rows) > 0: + in_reply_to = rows[0][0] + else: + want_twt = res msg_multilined = msg.replace('\u2028', '\n') msg_singlelined = ' '.join([i for i in msg.split('\u2028') if i.strip() != '']) @@ -1082,7 +1182,7 @@ def twt_line_to_mail(app, line, url, url_for_hash, nic mail_body += '\n' mail_body += f'{msg_multilined}\n' - return mail_body, twt_hash, twt_date + return mail_body, twt_hash, twt_date, want_twt def twt_lines_from_content(content, original_lines=False): @@ -1116,10 +1216,10 @@ if __name__ == '__main__': app = { 'config': config, - 'existing_twt_cache': None, } initialize_maildir(app) + initialize_sqlite(app) if args.edit: edit_twt_file(app) @@ -1130,14 +1230,16 @@ if __name__ == '__main__': elif args.last_seen: print('Feeds last seen at (times are local time), oldest first:') print() - show_lastseen(app, get_stamp_file('lastseen'), 'never') + show_lastseen(app, get_stamp_file(app, 'lastseen'), 'never') elif args.last_twt: print('Last twt in each feed (times are local time), oldest first:') print() - show_lastseen(app, get_stamp_file('lasttwts'), 'unknown') + show_lastseen(app, get_stamp_file(app, 'lasttwts'), 'unknown') elif args.debug_feed: debug_feed(args.debug_feed) elif args.fetch_context: fetch_context(app) else: compose(app, args.reply_to_this) + + close_sqlite(app) blob - c39e68bb215b9f9d9f45b4e0ae9531b8b04da2f2 file + tests/test_feed_metadata.py --- tests/test_feed_metadata.py +++ tests/test_feed_metadata.py @@ -34,7 +34,7 @@ def test_different_url_for_hash_and_different_nick(): lines = jenny.twt_lines_from_content(content) assert len(lines) == 1 - mail, _, _ = jenny.twt_line_to_mail( + mail, _, _, _ = jenny.twt_line_to_mail( APP, lines[0], configured_url, blob - 430fc3f86e7cc4c3bb8595b0da485771c187e35f file + tests/test_fetch_context.py --- tests/test_fetch_context.py +++ tests/test_fetch_context.py @@ -23,7 +23,7 @@ FEED_URL = 'https://zoinks.com' def test_not_yet_threaded(): - mail, _, _ = jenny.twt_line_to_mail( + mail, _, _, _ = jenny.twt_line_to_mail( APP, LINE_DATE + 'not a thread yet' + LINE_SUFFIX, FEED_URL, @@ -35,7 +35,7 @@ def test_not_yet_threaded(): def test_thread_without_mention(): - mail, _, _ = jenny.twt_line_to_mail( + mail, _, _, _ = jenny.twt_line_to_mail( APP, LINE_DATE + '(#1234567) hello' + LINE_SUFFIX, FEED_URL, @@ -51,7 +51,7 @@ def test_thread_without_mention(): def test_thread_mention(): - mail, _, _ = jenny.twt_line_to_mail( + mail, _, _, _ = jenny.twt_line_to_mail( APP, LINE_DATE + '(#1234567) @ I agree' + LINE_SUFFIX, FEED_URL, blob - 09eda7da7049fea11c1fde6113a2bbb0d9d25183 file + tests/test_generate_mail_body.py --- tests/test_generate_mail_body.py +++ tests/test_generate_mail_body.py @@ -25,7 +25,7 @@ FEED_URL = 'https://zoinks.com' def test_simple_body(): - mail, _, _ = jenny.twt_line_to_mail( + mail, _, _, _ = jenny.twt_line_to_mail( APP, '2020-12-04T23:19:55Z\thello world!', FEED_URL, @@ -45,7 +45,7 @@ hello world! def test_simple_body_with_space(): - mail, _, _ = jenny.twt_line_to_mail( + mail, _, _, _ = jenny.twt_line_to_mail( APP, '2020-12-04T23:19:55Z hello world!', FEED_URL, @@ -65,7 +65,7 @@ hello world! def test_multiline(): - mail, _, _ = jenny.twt_line_to_mail( + mail, _, _, _ = jenny.twt_line_to_mail( APP, '2020-12-04T23:19:55Z\thello world!\u2028\u2028and more stuff', FEED_URL, @@ -87,7 +87,7 @@ and more stuff def test_nick_desc(): - mail, _, _ = jenny.twt_line_to_mail( + mail, _, _, _ = jenny.twt_line_to_mail( APP, '2020-12-04T23:19:55Z\thello world!', FEED_URL, @@ -109,7 +109,7 @@ hello world! def test_mention(): this_app = deepcopy(APP) this_app['config']['mentions'] = set('@') - mail, _, _ = jenny.twt_line_to_mail( + mail, _, _, _ = jenny.twt_line_to_mail( this_app, '2020-12-04T23:19:55Z\thello @!', FEED_URL, @@ -132,7 +132,7 @@ def test_mention_custom_marker(): this_app = deepcopy(APP) this_app['config']['mentions'] = set('@') this_app['config']['mention_marker'] = '[MENTION]' - mail, _, _ = jenny.twt_line_to_mail( + mail, _, _, _ = jenny.twt_line_to_mail( this_app, '2020-12-04T23:19:55Z\thello @!', FEED_URL, @@ -152,7 +152,7 @@ hello @! def test_conv_tag_stripped(): - mail, _, _ = jenny.twt_line_to_mail( + mail, _, _, _ = jenny.twt_line_to_mail( APP, '2020-12-04T23:19:55Z\t(#inxgwfq) hello thread', FEED_URL, blob - f7adc42794d283dac84de87843227cc086c1a96a file + tests/test_reply_prefill.py --- tests/test_reply_prefill.py +++ tests/test_reply_prefill.py @@ -23,7 +23,7 @@ FEED_URL = 'https://zoinks.com' def test_not_yet_threaded(): - mail, _, _ = jenny.twt_line_to_mail( + mail, _, _, _ = jenny.twt_line_to_mail( APP, LINE_DATE + 'something something twt 🥳' + LINE_SUFFIX, FEED_URL, @@ -44,7 +44,7 @@ def test_not_yet_threaded(): def test_threaded_without_url_no_mention(): - mail, _, _ = jenny.twt_line_to_mail( + mail, _, _, _ = jenny.twt_line_to_mail( APP, LINE_DATE + '(#inxgwfq) my thread 🥳' + LINE_SUFFIX, FEED_URL, @@ -65,7 +65,7 @@ def test_threaded_without_url_no_mention(): def test_threaded_without_url_one_mention(): - mail, _, _ = jenny.twt_line_to_mail( + mail, _, _, _ = jenny.twt_line_to_mail( APP, LINE_DATE + '@ (#inxgwfq) my thread 🥳' + LINE_SUFFIX, FEED_URL, @@ -86,7 +86,7 @@ def test_threaded_without_url_one_mention(): def test_threaded_without_url_many_mentions_and_weird_spacing(): - mail, _, _ = jenny.twt_line_to_mail( + mail, _, _, _ = jenny.twt_line_to_mail( APP, LINE_DATE + ' @ (#inxgwfq) @ my thread 🥳' + LINE_SUFFIX, FEED_URL, @@ -107,7 +107,7 @@ def test_threaded_without_url_many_mentions_and_weird_ def test_not_yet_threaded_reply_to_this(): - mail, _, _ = jenny.twt_line_to_mail( + mail, _, _, _ = jenny.twt_line_to_mail( APP, LINE_DATE + 'something something twt 🥳' + LINE_SUFFIX, FEED_URL, @@ -128,7 +128,7 @@ def test_not_yet_threaded_reply_to_this(): def test_threaded_without_url_reply_to_this(): - mail, _, _ = jenny.twt_line_to_mail( + mail, _, _, _ = jenny.twt_line_to_mail( APP, LINE_DATE + '(#inxgwfq) something something twt 🥳' + LINE_SUFFIX, FEED_URL, @@ -149,7 +149,7 @@ def test_threaded_without_url_reply_to_this(): def test_dont_mention_self(): - mail, _, _ = jenny.twt_line_to_mail( + mail, _, _, _ = jenny.twt_line_to_mail( APP, LINE_DATE + 'something something twt 🥳' + LINE_SUFFIX, FEED_URL, blob - /dev/null file + tests/test_sqlite.py (mode 644) --- /dev/null +++ tests/test_sqlite.py @@ -0,0 +1,178 @@ +# "jenny" shall remain a simple Python script that you can just run. +# There shall not be the need for "pip install -e ." or something like +# that. Hence this little dance. +import importlib.machinery, importlib.util +loader = importlib.machinery.SourceFileLoader('jenny', './jenny') +spec = importlib.util.spec_from_loader(loader.name, loader) +jenny = importlib.util.module_from_spec(spec) +loader.exec_module(jenny) + + +# TODO: +# - Handle feeds with multiple URLs: a twt should be able to reference any of them. + +import os +import sqlite3 +import tempfile +import unittest + +TWT_0_TWT = '2020-12-04T23:19:55.0Z\thello world!' +TWT_0_URL = 'https://zoinks.com' +TWT_0_TIME = '2020-12-04T23:19:55.0Z' +TWT_0_FILENAME = '12345.54321_0.example.com' +TWT_0_HASH = '6sa4rqq' +TWT_0_NICK = 'cathy' +TWT_0_MAIL = '''\ +From: "cathy" +Subject: hello world! +Date: Fri, 04 Dec 2020 23:19:55 +0000 +Message-Id: <6sa4rqq@twtxt> +X-twtxt-feed-url: https://zoinks.com + +hello world! +''' + +TWT_1_TWT = '2020-12-05T00:07Z\t(replyto https://zoinks.com 2020-12-04T23:19:55.0Z) hi' +TWT_1_URL = 'https://zzz.com/twtxt.txt' +TWT_1_TIME = '2020-12-05T00:07Z' +TWT_1_FILENAME = '67899.54321_1.example.com' +TWT_1_HASH = 'thlkeia' +TWT_1_NICK = 'john' +# Without In-Reply-To +TWT_1_MAIL_UNRESOLVED = '''\ +From: "john" +Subject: (replyto https://zoinks.com 2020-12-04T23:19:55.0Z) hi +Date: Sat, 05 Dec 2020 00:07:00 +0000 +Message-Id: +X-twtxt-feed-url: https://zzz.com/twtxt.txt + +(replyto https://zoinks.com 2020-12-04T23:19:55.0Z) hi +''' +# With In-Reply-To +TWT_1_MAIL_RESOLVED = '''\ +From: "john" +Subject: (replyto https://zoinks.com 2020-12-04T23:19:55.0Z) hi +Date: Sat, 05 Dec 2020 00:07:00 +0000 +Message-Id: +X-twtxt-feed-url: https://zzz.com/twtxt.txt +In-Reply-To: <6sa4rqq@twtxt> + +(replyto https://zoinks.com 2020-12-04T23:19:55.0Z) hi +''' + +class TestSqlite(unittest.TestCase): + # Common test setup: create temporary directory for maildir and feed states. + def setUp(self): + self.dir = tempfile.TemporaryDirectory() + self.addCleanup(self.dir.cleanup) + self.app = { + 'config': { + 'feed_states_dir': f'{self.dir.name}/cache', + 'maildir_target': f'{self.dir.name}/Maildir', + 'mentions': set(), + 'mention_marker': ']', + }, + } + os.mkdir(self.app['config']['feed_states_dir']) + os.mkdir(self.app['config']['maildir_target']) + for subdir in ('cur', 'new', 'tmp'): + os.mkdir(self.app['config']['maildir_target'] + '/' + subdir) + self.sqlite_path = self.app['config']['feed_states_dir'] + '/db.sqlite' + + def test_no_mail_means_empty_tables(self): + jenny.initialize_sqlite(self.app) + jenny.close_sqlite(self.app) + con = sqlite3.connect(self.sqlite_path) + result = con.execute('select count(*) from have_twt') + count, = result.fetchone() + self.assertEqual(count, 0) + result = con.execute('select count(*) from want_twt') + count, = result.fetchone() + self.assertEqual(count, 0) + con.close() + + def test_deliver(self): + jenny.initialize_sqlite(self.app) + jenny.deliver_twt(self.app, TWT_0_TWT, TWT_0_URL, TWT_0_URL, TWT_0_NICK, TWT_0_NICK, filename=TWT_0_FILENAME) + # A second delivery should have no effect. + jenny.deliver_twt(self.app, TWT_0_TWT, TWT_0_URL, TWT_0_URL, TWT_0_NICK, TWT_0_NICK, filename=TWT_0_FILENAME) + result = self.app['sqlite_con'].execute('select * from have_twt') + have_twts = result.fetchall() + self.assertEqual(have_twts, [(TWT_0_HASH, TWT_0_URL, TWT_0_TIME)]) + result = self.app['sqlite_con'].execute('select count(*) from want_twt') + count, = result.fetchone() + self.assertEqual(count, 0) + jenny.close_sqlite(self.app) + + def test_want_twt(self): + # TWT_1 refers to TWT_0 by url and time, but TWT_0 is missing. So, we put + # an entry in want_twt. + # + # Then we add TWT_0, which should cause the row in want_twt to go away. + jenny.initialize_sqlite(self.app) + jenny.deliver_twt(self.app, TWT_1_TWT, TWT_1_URL, TWT_1_URL, TWT_1_NICK, TWT_1_NICK, filename=TWT_1_FILENAME) + result = self.app['sqlite_con'].execute('select * from have_twt') + have_twts = result.fetchall() + self.assertEqual(have_twts, [(TWT_1_HASH, TWT_1_URL, TWT_1_TIME)]) + result = self.app['sqlite_con'].execute('select * from want_twt') + want_twts = result.fetchall() + self.assertEqual(want_twts, [(TWT_0_URL, TWT_0_TIME, TWT_1_FILENAME)]) + + jenny.deliver_twt(self.app, TWT_0_TWT, TWT_0_URL, TWT_0_URL, TWT_0_NICK, TWT_0_NICK, filename=TWT_0_FILENAME) + + result = self.app['sqlite_con'].execute('select * from have_twt order by source_url') + have_twts = result.fetchall() + self.assertEqual(have_twts, [ + (TWT_0_HASH, TWT_0_URL, TWT_0_TIME), + (TWT_1_HASH, TWT_1_URL, TWT_1_TIME), + ]) + result = self.app['sqlite_con'].execute('select * from want_twt') + want_twts = result.fetchall() + self.assertEqual(want_twts, []) + + # Find the new filename for twt_1. It's the one that isn't + # TWT_0_FILENAME. + mail_names = tuple(name + for name in os.listdir(self.app['config']['maildir_target'] + '/new') + if name != TWT_0_FILENAME) + self.assertEqual(len(mail_names), 1) + twt_1_file = open(self.app['config']['maildir_target'] + '/new/' + mail_names[0]) + self.assertEqual(twt_1_file.read(), TWT_1_MAIL_RESOLVED) + twt_1_file.close(); + + jenny.close_sqlite(self.app) + + def test_already_have_target(self): + # Like the previous, but we add TWT_0 first. + jenny.initialize_sqlite(self.app) + jenny.deliver_twt(self.app, TWT_0_TWT, TWT_0_URL, TWT_0_URL, TWT_0_NICK, TWT_0_NICK, filename=TWT_0_FILENAME) + result = self.app['sqlite_con'].execute('select * from have_twt') + have_twts = result.fetchall() + self.assertEqual(have_twts, [(TWT_0_HASH, TWT_0_URL, TWT_0_TIME)]) + result = self.app['sqlite_con'].execute('select * from want_twt') + want_twts = result.fetchall() + self.assertEqual(want_twts, []) + + jenny.deliver_twt(self.app, TWT_1_TWT, TWT_1_URL, TWT_1_URL, TWT_1_NICK, TWT_1_NICK, filename=TWT_1_FILENAME) + + result = self.app['sqlite_con'].execute('select * from have_twt order by source_url') + have_twts = result.fetchall() + self.assertEqual(have_twts, [ + (TWT_0_HASH, TWT_0_URL, TWT_0_TIME), + (TWT_1_HASH, TWT_1_URL, TWT_1_TIME), + ]) + result = self.app['sqlite_con'].execute('select * from want_twt') + want_twts = result.fetchall() + self.assertEqual(want_twts, []) + + # Find the new filename for twt_1. It's the one that isn't + # TWT_0_FILENAME. + mail_names = tuple(name + for name in os.listdir(self.app['config']['maildir_target'] + '/new') + if name != TWT_0_FILENAME) + self.assertEqual(len(mail_names), 1) + twt_1_file = open(self.app['config']['maildir_target'] + '/new/' + mail_names[0]) + self.assertEqual(twt_1_file.read(), TWT_1_MAIL_RESOLVED) + twt_1_file.close(); + + jenny.close_sqlite(self.app)