split out requirements-{database,formats}.txt

requirements-database.txt

@@ -0,0 +1,7 @@
+# packages required for testing petl with databases
+
+cryptography 
+pymysql 
+SQLAlchemy>=1.3.6
+psycopg2-binary
+# PyMySQL==0.9.3

requirements-formats.txt

@@ -0,0 +1,13 @@
+Cython<0.29.21,>=0.29.13
+numpy<=1.19.2,>=1.16.4
+numexpr<=2.7.1,>=2.6.9
+intervaltree==3.0.2
+lxml==4.6.2
+openpyxl==2.6.2
+pandas<=1.1.2,>=0.24.2 ; python_version < '3.9'
+tables
+Whoosh==2.7.4
+xlrd==1.2.0
+xlwt==1.3.0
+fastavro>=0.24.2 ; python_version >= '3.4'
+fastavro==0.24.2 ; python_version < '3.0'

requirements-optional.txt

@@ -1,23 +1,7 @@
-Cython<0.29.21,>=0.29.13
-numpy<=1.19.2,>=1.16.4
-numexpr<=2.7.1,>=2.6.9
-intervaltree==3.0.2
-lxml==4.6.2
-openpyxl==2.6.2
-pandas<=1.1.2,>=0.24.2 ; python_version < '3.9'
-PyMySQL==0.9.3
-SQLAlchemy==1.3.6
-Whoosh==2.7.4
-xlrd==1.2.0
-xlwt==1.3.0
-fastavro>=0.24.2 ; python_version >= '3.4'
-fastavro==0.24.2 ; python_version < '3.0'
-smbprotocol>=1.0.1
-requests; python_version >= '3.4'
-fsspec>=0.7.4 ; python_version >= '3.4'
-aiohttp>=3.6.2 ; python_version >= '3.5.3'
-s3fs>=0.2.2 ; python_version >= '3.4'
 # packages bellow need complex local setup
-psycopg2<=2.8.6,>=2.8.3
-bcolz<=1.16.4,>=1.2.1
-tables<=3.6.1,>=3.5.2
+# throubleshooting: 
+# 1. define the following variable before running pip:
+# $ export DISABLE_BLOSC_AVX2=1
+# 2. pip install --prefer-binary bcolz
+blosc  ; python_version >= '3.7'
+bcolz  ; python_version >= '3.7'

tox.ini

@@ -25,7 +25,7 @@ deps =
     :preinstall1: numpy<=1.19.2,>=1.16.4
     :preinstall2: numexpr<=2.7.1,>=2.6.9
     -rrequirements-tests.txt
-    -rrequirements-optional.txt
+    -rrequirements-formats.txt
 
 [testenv:{py37,py38,py39}-docs]
 # build documentation under similar environment to readthedocs
@@ -48,12 +48,12 @@ commands =
 
 [testenv:remote]
 # Create test containers with the following commands:
-# docker run -it --name samba -p 139:139 -p 445:445 -d "dperson/samba" -p -u "user1;pass1" -s "public;/public-dir;yes;no;yes;all"
-# docker run -it --name sftp -p 22:22 -d atmoz/sftp user2:pass2:::public
+# docker run -it --name samba -p 139:139 -p 445:445 -d "dperson/samba" -p -u "petl;test" -s "public;/public-dir;yes;no;yes;all"
+# docker run -it --name sftp -p 22:22 -d atmoz/sftp petl:test:::public
 setenv =
     {[testenv]setenv}
-    PETL_TEST_SMB=smb://WORKGROUP;user1:pass1@localhost/public/
-    PETL_TEST_SFTP=sftp://user1:pass1@localhost/public/
+    PETL_TEST_SMB=smb://WORKGROUP;petl:test@localhost/public/
+    PETL_TEST_SFTP=sftp://petl:test@localhost/public/
 commands =
     nosetests -v petl --with-coverage --cover-package=petl
 deps =
@@ -69,12 +69,8 @@ setenv =
 commands =
     nosetests -v petl --with-coverage --cover-package=petl
 deps =
-    cryptography
-    pymysql
-    SQLAlchemy
-    psycopg2-binary
-    # mysqlclient
     -rrequirements-tests.txt
+    -rrequirements-database.txt
 
 [testenv:mysqldb]
 basepython = python2.7