Deprecate wget/WARC-based crawler approach

PR 81 implemented a new crawler approach based on wpull. This change deprecates the old approach based on wget crawling into an intermediate WARC file.
cfpb · Jul 2, 2024 · cbfb66c · cbfb66c
1 parent 8bd065d
commit cbfb66c
Show file tree

Hide file tree

Showing 8 changed files with 1 addition and 522 deletions.
diff --git a/.gitignore b/.gitignore
@@ -31,12 +31,6 @@ _site/
 *.sqlite3
 !sample/sample.sqlite3
 
-# Crawl files, except for the sample crawl #
-############################################
-*.warc
-*.warc.gz
-!sample/crawl.warc.gz
-
 # OS generated files #
 ######################
 .DS_Store

diff --git a/crawler/management/commands/warc_to_csv.py b/crawler/management/commands/warc_to_csv.py
diff --git a/crawler/management/commands/warc_to_db.py b/crawler/management/commands/warc_to_db.py
diff --git a/crawler/reader.py b/crawler/reader.py
diff --git a/crawler/wpull_plugin.py b/crawler/wpull_plugin.py
@@ -69,7 +69,7 @@ def deactivate(self):
         self.db_writer.analyze()
 
     def init_db(self):
-        db_alias = "warc_to_db"
+        db_alias = "crawler"
 
         connections.databases[db_alias] = {
             "ENGINE": "django.db.backends.sqlite3",