Ausgabe der neuen DB Einträge

venv/lib/python3.9/site-packages/scrapy/commands/check.py

@@ -0,0 +1,96 @@
+import time
+from collections import defaultdict
+from unittest import TextTestRunner, TextTestResult as _TextTestResult
+
+from scrapy.commands import ScrapyCommand
+from scrapy.contracts import ContractsManager
+from scrapy.utils.misc import load_object, set_environ
+from scrapy.utils.conf import build_component_list
+
+
+class TextTestResult(_TextTestResult):
+    def printSummary(self, start, stop):
+        write = self.stream.write
+        writeln = self.stream.writeln
+
+        run = self.testsRun
+        plural = "s" if run != 1 else ""
+
+        writeln(self.separator2)
+        writeln(f"Ran {run} contract{plural} in {stop - start:.3f}s")
+        writeln()
+
+        infos = []
+        if not self.wasSuccessful():
+            write("FAILED")
+            failed, errored = map(len, (self.failures, self.errors))
+            if failed:
+                infos.append(f"failures={failed}")
+            if errored:
+                infos.append(f"errors={errored}")
+        else:
+            write("OK")
+
+        if infos:
+            writeln(f" ({', '.join(infos)})")
+        else:
+            write("\n")
+
+
+class Command(ScrapyCommand):
+    requires_project = True
+    default_settings = {'LOG_ENABLED': False}
+
+    def syntax(self):
+        return "[options] <spider>"
+
+    def short_desc(self):
+        return "Check spider contracts"
+
+    def add_options(self, parser):
+        ScrapyCommand.add_options(self, parser)
+        parser.add_option("-l", "--list", dest="list", action="store_true",
+                          help="only list contracts, without checking them")
+        parser.add_option("-v", "--verbose", dest="verbose", default=False, action='store_true',
+                          help="print contract tests for all spiders")
+
+    def run(self, args, opts):
+        # load contracts
+        contracts = build_component_list(self.settings.getwithbase('SPIDER_CONTRACTS'))
+        conman = ContractsManager(load_object(c) for c in contracts)
+        runner = TextTestRunner(verbosity=2 if opts.verbose else 1)
+        result = TextTestResult(runner.stream, runner.descriptions, runner.verbosity)
+
+        # contract requests
+        contract_reqs = defaultdict(list)
+
+        spider_loader = self.crawler_process.spider_loader
+
+        with set_environ(SCRAPY_CHECK='true'):
+            for spidername in args or spider_loader.list():
+                spidercls = spider_loader.load(spidername)
+                spidercls.start_requests = lambda s: conman.from_spider(s, result)
+
+                tested_methods = conman.tested_methods_from_spidercls(spidercls)
+                if opts.list:
+                    for method in tested_methods:
+                        contract_reqs[spidercls.name].append(method)
+                elif tested_methods:
+                    self.crawler_process.crawl(spidercls)
+
+            # start checks
+            if opts.list:
+                for spider, methods in sorted(contract_reqs.items()):
+                    if not methods and not opts.verbose:
+                        continue
+                    print(spider)
+                    for method in sorted(methods):
+                        print(f'  * {method}')
+            else:
+                start = time.time()
+                self.crawler_process.start()
+                stop = time.time()
+
+                result.printErrors()
+                result.printSummary(start, stop)
+                self.exitcode = int(not result.wasSuccessful())