Ausgabe der neuen DB Einträge

This commit is contained in:
hubobel 2022-01-02 21:50:48 +01:00
parent bad48e1627
commit cfbbb9ee3d
2399 changed files with 843193 additions and 43 deletions

View file

@ -0,0 +1,33 @@
import unittest, tempfile, shutil
class QueuelibTestCase(unittest.TestCase):
def setUp(self):
self.tmpdir = tempfile.mkdtemp(prefix="queuelib-tests-")
self.qpath = self.mktemp()
self.qdir = self.mkdtemp()
def tearDown(self):
shutil.rmtree(self.tmpdir)
def mktemp(self):
return tempfile.mktemp(dir=self.tmpdir)
def mkdtemp(self):
return tempfile.mkdtemp(dir=self.tmpdir)
def track_closed(cls):
"""Wraps a queue class to track down if close() method was called"""
class TrackingClosed(cls):
def __init__(self, *a, **kw):
super(TrackingClosed, self).__init__(*a, **kw)
self.closed = False
def close(self):
super(TrackingClosed, self).close()
self.closed = True
return TrackingClosed

View file

@ -0,0 +1,184 @@
import os
from queuelib.pqueue import PriorityQueue
from queuelib.queue import (
FifoMemoryQueue, LifoMemoryQueue, FifoDiskQueue, LifoDiskQueue,
FifoSQLiteQueue, LifoSQLiteQueue,
)
from queuelib.tests import (QueuelibTestCase, track_closed)
# hack to prevent py.test from discovering base test class
class base:
class PQueueTestBase(QueuelibTestCase):
def setUp(self):
QueuelibTestCase.setUp(self)
self.q = PriorityQueue(self.qfactory)
def qfactory(self, prio):
raise NotImplementedError
def test_len_nonzero(self):
assert not self.q
self.assertEqual(len(self.q), 0)
self.q.push(b'a', 3)
assert self.q
self.q.push(b'b', 1)
self.q.push(b'c', 2)
self.q.push(b'd', 1)
self.assertEqual(len(self.q), 4)
self.q.pop()
self.q.pop()
self.q.pop()
self.q.pop()
assert not self.q
self.assertEqual(len(self.q), 0)
def test_close(self):
self.q.push(b'a', 3)
self.q.push(b'b', 1)
self.q.push(b'c', 2)
self.q.push(b'd', 1)
iqueues = self.q.queues.values()
self.assertEqual(sorted(self.q.close()), [1, 2, 3])
assert all(q.closed for q in iqueues)
def test_close_return_active(self):
self.q.push(b'b', 1)
self.q.push(b'c', 2)
self.q.push(b'a', 3)
self.q.pop()
self.assertEqual(sorted(self.q.close()), [2, 3])
def test_popped_internal_queues_closed(self):
self.q.push(b'a', 3)
self.q.push(b'b', 1)
self.q.push(b'c', 2)
p1queue = self.q.queues[1]
self.assertEqual(self.q.pop(), b'b')
self.q.close()
assert p1queue.closed
class FifoTestMixin(object):
def test_push_pop_noprio(self):
self.q.push(b'a')
self.q.push(b'b')
self.q.push(b'c')
self.assertEqual(self.q.pop(), b'a')
self.assertEqual(self.q.pop(), b'b')
self.assertEqual(self.q.pop(), b'c')
self.assertEqual(self.q.pop(), None)
def test_push_pop_prio(self):
self.q.push(b'a', 3)
self.q.push(b'b', 1)
self.q.push(b'c', 2)
self.q.push(b'd', 1)
self.assertEqual(self.q.pop(), b'b')
self.assertEqual(self.q.pop(), b'd')
self.assertEqual(self.q.pop(), b'c')
self.assertEqual(self.q.pop(), b'a')
self.assertEqual(self.q.pop(), None)
class LifoTestMixin(object):
def test_push_pop_noprio(self):
self.q.push(b'a')
self.q.push(b'b')
self.q.push(b'c')
self.assertEqual(self.q.pop(), b'c')
self.assertEqual(self.q.pop(), b'b')
self.assertEqual(self.q.pop(), b'a')
self.assertEqual(self.q.pop(), None)
def test_push_pop_prio(self):
self.q.push(b'a', 3)
self.q.push(b'b', 1)
self.q.push(b'c', 2)
self.q.push(b'd', 1)
self.assertEqual(self.q.pop(), b'd')
self.assertEqual(self.q.pop(), b'b')
self.assertEqual(self.q.pop(), b'c')
self.assertEqual(self.q.pop(), b'a')
self.assertEqual(self.q.pop(), None)
class FifoMemoryPriorityQueueTest(FifoTestMixin, base.PQueueTestBase):
def qfactory(self, prio):
return track_closed(FifoMemoryQueue)()
class LifoMemoryPriorityQueueTest(LifoTestMixin, base.PQueueTestBase):
def qfactory(self, prio):
return track_closed(LifoMemoryQueue)()
class DiskTestMixin(object):
def test_nonserializable_object_one(self):
self.assertRaises(TypeError, self.q.push, lambda x: x, 0)
self.assertEqual(self.q.close(), [])
def test_nonserializable_object_many_close(self):
self.q.push(b'a', 3)
self.q.push(b'b', 1)
self.assertRaises(TypeError, self.q.push, lambda x: x, 0)
self.q.push(b'c', 2)
self.assertEqual(self.q.pop(), b'b')
self.assertEqual(sorted(self.q.close()), [2, 3])
def test_nonserializable_object_many_pop(self):
self.q.push(b'a', 3)
self.q.push(b'b', 1)
self.assertRaises(TypeError, self.q.push, lambda x: x, 0)
self.q.push(b'c', 2)
self.assertEqual(self.q.pop(), b'b')
self.assertEqual(self.q.pop(), b'c')
self.assertEqual(self.q.pop(), b'a')
self.assertEqual(self.q.pop(), None)
self.assertEqual(self.q.close(), [])
def test_reopen_with_prio(self):
q1 = PriorityQueue(self.qfactory)
q1.push(b'a', 3)
q1.push(b'b', 1)
q1.push(b'c', 2)
active = q1.close()
q2 = PriorityQueue(self.qfactory, startprios=active)
self.assertEqual(q2.pop(), b'b')
self.assertEqual(q2.pop(), b'c')
self.assertEqual(q2.pop(), b'a')
self.assertEqual(q2.close(), [])
class FifoDiskPriorityQueueTest(FifoTestMixin, DiskTestMixin, base.PQueueTestBase):
def qfactory(self, prio):
path = os.path.join(self.qdir, str(prio))
return track_closed(FifoDiskQueue)(path)
class LifoDiskPriorityQueueTest(LifoTestMixin, DiskTestMixin, base.PQueueTestBase):
def qfactory(self, prio):
path = os.path.join(self.qdir, str(prio))
return track_closed(LifoDiskQueue)(path)
class FifoSQLitePriorityQueueTest(FifoTestMixin, DiskTestMixin, base.PQueueTestBase):
def qfactory(self, prio):
path = os.path.join(self.qdir, str(prio))
return track_closed(FifoSQLiteQueue)(path)
class LifoSQLitePriorityQueueTest(LifoTestMixin, DiskTestMixin, base.PQueueTestBase):
def qfactory(self, prio):
path = os.path.join(self.qdir, str(prio))
return track_closed(LifoSQLiteQueue)(path)

View file

@ -0,0 +1,245 @@
import os
import glob
import pytest
from queuelib.queue import (
FifoMemoryQueue, LifoMemoryQueue, FifoDiskQueue, LifoDiskQueue,
FifoSQLiteQueue, LifoSQLiteQueue,
)
from queuelib.tests import QueuelibTestCase
class BaseQueueTest(object):
def queue(self):
return NotImplementedError()
def test_empty(self):
"""Empty queue test"""
q = self.queue()
assert q.pop() is None
def test_single_pushpop(self):
q = self.queue()
q.push(b'a')
assert q.pop() == b'a'
def test_binary_element(self):
elem = (
b'\x80\x02}q\x01(U\x04bodyq\x02U\x00U\t_encodingq\x03U\x05utf-'
b'8q\x04U\x07cookiesq\x05}q\x06U\x04metaq\x07}q\x08U\x07header'
b'sq\t}U\x03urlq\nX\x15\x00\x00\x00file:///tmp/tmphDJYsgU\x0bd'
b'ont_filterq\x0b\x89U\x08priorityq\x0cK\x00U\x08callbackq\rNU'
b'\x06methodq\x0eU\x03GETq\x0fU\x07errbackq\x10Nu.'
)
q = self.queue()
q.push(elem)
assert q.pop() == elem
def test_len(self):
q = self.queue()
self.assertEqual(len(q), 0)
q.push(b'a')
self.assertEqual(len(q), 1)
q.push(b'b')
q.push(b'c')
self.assertEqual(len(q), 3)
q.pop()
q.pop()
q.pop()
self.assertEqual(len(q), 0)
class FifoTestMixin(BaseQueueTest):
def test_push_pop1(self):
"""Basic push/pop test"""
q = self.queue()
q.push(b'a')
q.push(b'b')
q.push(b'c')
self.assertEqual(q.pop(), b'a')
self.assertEqual(q.pop(), b'b')
self.assertEqual(q.pop(), b'c')
self.assertEqual(q.pop(), None)
def test_push_pop2(self):
"""Test interleaved push and pops"""
q = self.queue()
q.push(b'a')
q.push(b'b')
q.push(b'c')
q.push(b'd')
self.assertEqual(q.pop(), b'a')
self.assertEqual(q.pop(), b'b')
q.push(b'e')
self.assertEqual(q.pop(), b'c')
self.assertEqual(q.pop(), b'd')
self.assertEqual(q.pop(), b'e')
class LifoTestMixin(BaseQueueTest):
def test_push_pop1(self):
"""Basic push/pop test"""
q = self.queue()
q.push(b'a')
q.push(b'b')
q.push(b'c')
self.assertEqual(q.pop(), b'c')
self.assertEqual(q.pop(), b'b')
self.assertEqual(q.pop(), b'a')
self.assertEqual(q.pop(), None)
def test_push_pop2(self):
"""Test interleaved push and pops"""
q = self.queue()
q.push(b'a')
q.push(b'b')
q.push(b'c')
q.push(b'd')
self.assertEqual(q.pop(), b'd')
self.assertEqual(q.pop(), b'c')
q.push(b'e')
self.assertEqual(q.pop(), b'e')
self.assertEqual(q.pop(), b'b')
self.assertEqual(q.pop(), b'a')
class PersistentTestMixin(object):
chunksize = 100000
@pytest.mark.xfail(reason="Reenable once Scrapy.squeues stop"
" extending from queuelib testsuite")
def test_non_bytes_raises_typeerror(self):
q = self.queue()
self.assertRaises(TypeError, q.push, 0)
self.assertRaises(TypeError, q.push, u'')
self.assertRaises(TypeError, q.push, None)
self.assertRaises(TypeError, q.push, lambda x: x)
def test_text_in_windows(self):
e1 = b'\r\n'
q = self.queue()
q.push(e1)
q.close()
q = self.queue()
e2 = q.pop()
self.assertEqual(e1, e2)
def test_close_open(self):
"""Test closing and re-opening keeps state"""
q = self.queue()
q.push(b'a')
q.push(b'b')
q.push(b'c')
q.push(b'd')
q.pop()
q.pop()
q.close()
del q
q = self.queue()
self.assertEqual(len(q), 2)
q.push(b'e')
q.pop()
q.pop()
q.close()
del q
q = self.queue()
assert q.pop() is not None
self.assertEqual(len(q), 0)
def test_cleanup(self):
"""Test queue dir is removed if queue is empty"""
q = self.queue()
values = [b'0', b'1', b'2', b'3', b'4']
assert os.path.exists(self.qpath)
for x in values:
q.push(x)
for x in values:
q.pop()
q.close()
assert not os.path.exists(self.qpath)
class FifoMemoryQueueTest(FifoTestMixin, QueuelibTestCase):
def queue(self):
return FifoMemoryQueue()
class LifoMemoryQueueTest(LifoTestMixin, QueuelibTestCase):
def queue(self):
return LifoMemoryQueue()
class FifoDiskQueueTest(FifoTestMixin, PersistentTestMixin, QueuelibTestCase):
def queue(self):
return FifoDiskQueue(self.qpath, chunksize=self.chunksize)
def test_chunks(self):
"""Test chunks are created and removed"""
values = [b'0', b'1', b'2', b'3', b'4']
q = self.queue()
for x in values:
q.push(x)
chunks = glob.glob(os.path.join(self.qpath, 'q*'))
self.assertEqual(len(chunks), 5 // self.chunksize + 1)
for x in values:
q.pop()
chunks = glob.glob(os.path.join(self.qpath, 'q*'))
self.assertEqual(len(chunks), 1)
class ChunkSize1FifoDiskQueueTest(FifoDiskQueueTest):
chunksize = 1
class ChunkSize2FifoDiskQueueTest(FifoDiskQueueTest):
chunksize = 2
class ChunkSize3FifoDiskQueueTest(FifoDiskQueueTest):
chunksize = 3
class ChunkSize4FifoDiskQueueTest(FifoDiskQueueTest):
chunksize = 4
class LifoDiskQueueTest(LifoTestMixin, PersistentTestMixin, QueuelibTestCase):
def queue(self):
return LifoDiskQueue(self.qpath)
def test_file_size_shrinks(self):
"""Test size of queue file shrinks when popping items"""
q = self.queue()
q.push(b'a')
q.push(b'b')
q.close()
size = os.path.getsize(self.qpath)
q = self.queue()
q.pop()
q.close()
assert os.path.getsize(self.qpath), size
class FifoSQLiteQueueTest(FifoTestMixin, PersistentTestMixin, QueuelibTestCase):
def queue(self):
return FifoSQLiteQueue(self.qpath)
class LifoSQLiteQueueTest(LifoTestMixin, PersistentTestMixin, QueuelibTestCase):
def queue(self):
return LifoSQLiteQueue(self.qpath)

View file

@ -0,0 +1,144 @@
import os
from queuelib.rrqueue import RoundRobinQueue
from queuelib.queue import (
FifoMemoryQueue, LifoMemoryQueue, FifoDiskQueue, LifoDiskQueue,
FifoSQLiteQueue, LifoSQLiteQueue,
)
from queuelib.tests import (QueuelibTestCase, track_closed)
# hack to prevent py.test from discovering base test class
class base:
class RRQueueTestBase(QueuelibTestCase):
def setUp(self):
QueuelibTestCase.setUp(self)
self.q = RoundRobinQueue(self.qfactory)
def qfactory(self, key):
raise NotImplementedError
def test_len_nonzero(self):
assert not self.q
self.assertEqual(len(self.q), 0)
self.q.push(b'a', '3')
assert self.q
self.q.push(b'b', '1')
self.q.push(b'c', '2')
self.q.push(b'd', '1')
self.assertEqual(len(self.q), 4)
self.q.pop()
self.q.pop()
self.q.pop()
self.q.pop()
assert not self.q
self.assertEqual(len(self.q), 0)
def test_close(self):
self.q.push(b'a', '3')
self.q.push(b'b', '1')
self.q.push(b'c', '2')
self.q.push(b'd', '1')
iqueues = self.q.queues.values()
self.assertEqual(sorted(self.q.close()), ['1', '2', '3'])
assert all(q.closed for q in iqueues)
def test_close_return_active(self):
self.q.push(b'b', '1')
self.q.push(b'c', '2')
self.q.push(b'a', '3')
self.q.pop()
self.assertEqual(sorted(self.q.close()), ['2', '3'])
class FifoTestMixin(object):
def test_push_pop_key(self):
self.q.push(b'a', '1')
self.q.push(b'b', '1')
self.q.push(b'c', '2')
self.q.push(b'd', '2')
self.assertEqual(self.q.pop(), b'a')
self.assertEqual(self.q.pop(), b'c')
self.assertEqual(self.q.pop(), b'b')
self.assertEqual(self.q.pop(), b'd')
self.assertEqual(self.q.pop(), None)
class LifoTestMixin(object):
def test_push_pop_key(self):
self.q.push(b'a', '1')
self.q.push(b'b', '1')
self.q.push(b'c', '2')
self.q.push(b'd', '2')
self.assertEqual(self.q.pop(), b'b')
self.assertEqual(self.q.pop(), b'd')
self.assertEqual(self.q.pop(), b'a')
self.assertEqual(self.q.pop(), b'c')
self.assertEqual(self.q.pop(), None)
class FifoMemoryRRQueueTest(FifoTestMixin, base.RRQueueTestBase):
def qfactory(self, key):
return track_closed(FifoMemoryQueue)()
class LifoMemoryRRQueueTest(LifoTestMixin, base.RRQueueTestBase):
def qfactory(self, key):
return track_closed(LifoMemoryQueue)()
class DiskTestMixin(object):
def test_nonserializable_object_one(self):
self.assertRaises(TypeError, self.q.push, lambda x: x, '0')
self.assertEqual(self.q.close(), [])
def test_nonserializable_object_many_close(self):
self.q.push(b'a', '3')
self.q.push(b'b', '1')
self.assertRaises(TypeError, self.q.push, lambda x: x, '0')
self.q.push(b'c', '2')
self.assertEqual(self.q.pop(), b'a')
self.assertEqual(sorted(self.q.close()), ['1', '2'])
def test_nonserializable_object_many_pop(self):
self.q.push(b'a', '3')
self.q.push(b'b', '1')
self.assertRaises(TypeError, self.q.push, lambda x: x, '0')
self.q.push(b'c', '2')
self.assertEqual(self.q.pop(), b'a')
self.assertEqual(self.q.pop(), b'b')
self.assertEqual(self.q.pop(), b'c')
self.assertEqual(self.q.pop(), None)
self.assertEqual(self.q.close(), [])
class FifoDiskRRQueueTest(FifoTestMixin, DiskTestMixin, base.RRQueueTestBase):
def qfactory(self, key):
path = os.path.join(self.qdir, str(key))
return track_closed(FifoDiskQueue)(path)
class LifoDiskRRQueueTest(LifoTestMixin, DiskTestMixin, base.RRQueueTestBase):
def qfactory(self, key):
path = os.path.join(self.qdir, str(key))
return track_closed(LifoDiskQueue)(path)
class FifoSQLiteRRQueueTest(FifoTestMixin, DiskTestMixin, base.RRQueueTestBase):
def qfactory(self, key):
path = os.path.join(self.qdir, str(key))
return track_closed(FifoSQLiteQueue)(path)
class LifoSQLiteRRQueueTest(LifoTestMixin, DiskTestMixin, base.RRQueueTestBase):
def qfactory(self, key):
path = os.path.join(self.qdir, str(key))
return track_closed(LifoSQLiteQueue)(path)