Add test for piece resume and fix bugs discovered

diff --git a/src/disk_jumble/db.py b/src/disk_jumble/db.py
index 8bb49e35ee9244f3c0810d57ef5896dae89273a0..ee93efce1548a103170a094b0fe031589c6f5c00 100644 (file)
--- a/src/disk_jumble/db.py
+++ b/src/disk_jumble/db.py
@@ -101,7 +101,7 @@ class Wrapper:
                                                natural left join diskjumble.disk
                                                left join diskjumble.slab on (
                                                        c.disk_id = slab.disk_id
-                                                       and upper(c.disk_sectors) <@ slab.disk_sectors
+                                                       and upper(c.disk_sectors) <@ int8range(lower(slab.disk_sectors), upper(slab.disk_sectors), '[]')
                                                )
                                        where seq >= all (select seq from diskjumble.verify_piece_content where verify_id = p.verify_id)
                                )
@@ -113,11 +113,11 @@ class Wrapper:
                                natural left join diskjumble.disk
                                left join incomplete_edge on
                                        incomplete_edge.entity_id = slab.entity_id
-                                       and incomplete_edge.end_off %% sector_size = 0
                                        and incomplete_edge.end_off <@ int8range(
                                                slab.entity_offset,
                                                slab.entity_offset + (upper(disk_sectors) - lower(disk_sectors)) * sector_size
                                        )
+                                       and (incomplete_edge.end_off - slab.entity_offset) %% sector_size = 0
                        where disk_id = %s
                        order by slab.entity_id, entity_offset, slab_id
                        ;

diff --git a/src/disk_jumble/nettle.py b/src/disk_jumble/nettle.py
index c432d1fcc08ef83f5175b520dc6ff2ed21fcb8d6..dfabbcac136a9f0dfe492196b5966af0f29b8348 100644 (file)
--- a/src/disk_jumble/nettle.py
+++ b/src/disk_jumble/nettle.py
@@ -30,10 +30,10 @@ class Sha1Hasher(_Sha1Defs):
                @classmethod
                def deserialize(cls, data):
                        return cls(
-                               cls.StateArr(*data["state"]),
+                               _Sha1Defs._StateArr(*data["state"]),
                                data["count"],
                                data["index"],
-                               cls.BlockArr(*data["block"]),
+                               _Sha1Defs._BlockArr(*data["block"]),
                        )
 
                def serialize(self):

diff --git a/src/disk_jumble/tests/test_verify.py b/src/disk_jumble/tests/test_verify.py
index 8443bcefa918b4f93b547b390d18099b056a78a6..d3d413347ed558a963cea33b9317c06f6f02e433 100644 (file)
--- a/src/disk_jumble/tests/test_verify.py
+++ b/src/disk_jumble/tests/test_verify.py
@@ -30,8 +30,8 @@ from disk_jumble.verify import do_verify
 _BUF_SIZE = 16 * 1024 ** 2  # in bytes
 
 
-def _random_file(size: int, rand_src: Random) -> tempfile.NamedTemporaryFile:
-       f = tempfile.NamedTemporaryFile(buffering = _BUF_SIZE)
+def _random_file(size: int, rand_src: Random, on_disk: bool) -> tempfile.NamedTemporaryFile:
+       f = tempfile.NamedTemporaryFile(buffering = _BUF_SIZE) if on_disk else io.BytesIO()
        try:
                while f.tell() < size:
                        write_size = min(size - f.tell(), _BUF_SIZE)
@@ -52,7 +52,7 @@ class Tests(unittest.TestCase):
 
                torrent_len = 3 * piece_size
                disk = self._write_disk(sector_size, torrent_len // sector_size)
-               with _random_file(torrent_len, Random(0)) as torrent_file:
+               with _random_file(torrent_len, Random(0), on_disk = False) as torrent_file:
                        torrent = _Torrent(torrent_file, piece_size)
                        torrent_file.seek(0)
                        self._write_torrent(torrent)
@@ -62,7 +62,7 @@ class Tests(unittest.TestCase):
                                        (disk.id, NumericRange(0, disk.sector_count), torrent.info_hash)
                                )
 
-                               do_verify(self._conn, disk.id, torrent_file, read_size, 1)
+                               do_verify(self._conn, disk.id, torrent_file, read_size, read_tries = 1)
 
                                cursor.execute("select * from diskjumble.verify_pass;")
                                self.assertEqual(cursor.rowcount, 1)
@@ -76,6 +76,62 @@ class Tests(unittest.TestCase):
        def test_basic_fresh_verify_large_read_size(self):
                self._basic_fresh_verify_helper(128)
 
+       def test_resume_fragmentation_unaligned_end(self):
+               """
+               Test a run where a cached hash state is used, a piece is split on disk, and the end of the torrent isn't
+               sector-aligned.
+               """
+               read_size = 8
+               piece_size = 64
+
+               other_disk = self._write_disk(16, 1)
+               disk = self._write_disk(32, 3)
+               with _random_file(piece_size, Random(0), on_disk = False) as torrent_file:
+                       torrent = _Torrent(torrent_file, piece_size)
+                       torrent_file.seek(0)
+                       self._write_torrent(torrent)
+                       with self._conn.cursor() as cursor:
+                               cursor.executemany(
+                                       "insert into diskjumble.slab values (default, %s, %s, %s, %s, null)",
+                                       [
+                                               (other_disk.id, NumericRange(0, 1), torrent.info_hash, 0),
+                                               (disk.id, NumericRange(0, 1), torrent.info_hash, other_disk.sector_size),
+                                               (disk.id, NumericRange(2, 3), torrent.info_hash, other_disk.sector_size + disk.sector_size),
+                                       ]
+                               )
+
+                               # Prepare the saved hasher state by running a verify
+                               do_verify(self._conn, other_disk.id, torrent_file, read_size, read_tries = 1)
+                               torrent_file.seek(0)
+
+                               cursor.execute("select count(*) from diskjumble.verify_piece_incomplete;")
+                               [(row_count,)] = cursor.fetchall()
+                               self.assertEqual(row_count, 1)
+
+                               disk_file = io.BytesIO()
+                               torrent_file.seek(other_disk.sector_size)
+                               disk_file.write(torrent_file.read(disk.sector_size))
+                               disk_file.seek(disk_file.tell() + disk.sector_size)
+                               disk_file.write(torrent_file.read())
+                               disk_file.seek(0)
+                               do_verify(self._conn, disk.id, disk_file, read_size, read_tries = 1)
+
+                               # Check that there are no verify pieces in the database.  Because of integrity constraints, this also
+                               # guarantees there aren't any stray saved hasher states, failed verifies, or piece contents.
+                               cursor.execute("select count(*) from diskjumble.verify_piece;")
+                               [(row_count,)] = cursor.fetchall()
+                               self.assertEqual(row_count, 0)
+
+                               cursor.execute("select disk_id, disk_sectors from diskjumble.verify_pass;")
+                               self.assertEqual(
+                                       cursor.fetchall(),
+                                       [(other_disk.id, NumericRange(0, 1)), (disk.id, NumericRange(0, 1)), (disk.id, NumericRange(2, 3))]
+                               )
+
+       # TODO ignore useless hasher state
+
+       # TODO read errors
+
        def _write_torrent(self, torrent: "_Torrent") -> None:
                with self._conn.cursor() as cursor:
                        cursor.execute("insert into bittorrent.torrent_info values (%s);", (torrent.info,))