4.1.2 (#39)

* add writting sector error during importation

* move stdout message to logs

* update github actions

* improve failure on node reading logic

.github/workflows/build.yml

@@ -30,7 +30,7 @@ jobs:
         run: echo ${{ steps.version.outputs.version }}
 
       - name: 'Tar files'
-        run: tar czvf ldb-${{ steps.version.outputs.version }}-amd64.tar.gz ldb libldb.so LICENSE
+        run: tar czvf ldb-${{ steps.version.outputs.version }}-amd64.tar.gz ldb libldb.so
 
       - name: Upload a Build Artifact
         uses: actions/upload-artifact@v3

.github/workflows/release.yml

@@ -28,7 +28,6 @@ jobs:
           mkdir -p ./artifacts/
           cp ldb ./artifacts/ldb
           cp libldb.so ./artifacts/libldb.so
-          cp LICENSE ./artifacts/LICENSE
           echo "Produced artifact at ${PWD}/artifacts/ldb"
 
       - name: 'Tar files'

src/hex.c

@@ -30,7 +30,7 @@
   * @see https://github.com/scanoss/ldb/blob/master/src/hex.c
   */
 #include "ldb.h"
-
+#include "logger.h"
 /**
  * @brief Prints a hexdump of 'len' bytes from 'data' organized 'width' columns
  * 
@@ -176,7 +176,12 @@ void ldb_uint32_write(FILE *ldb_sector, uint32_t value)
 uint32_t ldb_uint32_read(FILE *ldb_sector)
 {
 	uint32_t out = 0;
-	if (!fread((uint8_t*)&out, 1, 4, ldb_sector)) printf("Warning: cannot read LDB sector\n");
+	if (!fread((uint8_t*)&out, 1, 4, ldb_sector))
+	{
+		log_debug("Warning: cannot read LDB sector\n");
+		ldb_read_failure = true;
+
+	}
 	return out;
 }
 
@@ -189,7 +194,11 @@ uint32_t ldb_uint32_read(FILE *ldb_sector)
 uint64_t ldb_uint40_read(FILE *ldb_sector)
 {
 	uint64_t out = 0;
-	if (!fread((uint8_t*)&out, 1, 5, ldb_sector)) printf("Warning: cannot read LDB sector\n");
+	if (!fread((uint8_t*)&out, 1, 5, ldb_sector))
+	{
+		log_debug("Warning: cannot read LDB sector\n");
+		ldb_read_failure = true;
+	}
 	return out;
 }
 
@@ -202,7 +211,11 @@ uint64_t ldb_uint40_read(FILE *ldb_sector)
 uint16_t ldb_uint16_read(FILE *ldb_sector)
 {
 	uint16_t out = 0;
-	if (!fread((uint8_t*)&out, 1, 2, ldb_sector)) printf("Warning: cannot read LDB sector\n");
+	if (!fread((uint8_t*)&out, 1, 2, ldb_sector))
+	{ 
+		log_debug("Warning: cannot read LDB sector\n");
+		ldb_read_failure = true;
+	}
 	return out;
 }
 

src/import.c

@@ -536,10 +536,6 @@ int ldb_import_csv(ldb_importation_config_t * job)
 		return -1;
 	}
 
-	char *line = NULL;
-	size_t len = 0;
-	ssize_t lineln;
-
 	/* A CSV line should contain at least an MD5, a comma separator per field and a LF */
 	int min_line_size = 2 * MD5_LEN + (skip_csv_check > 0 ? 0 : job->opt.params.csv_fields);
 
@@ -621,18 +617,36 @@ int ldb_import_csv(ldb_importation_config_t * job)
 	csv_sort(job);
 	int line_number = 0;
 	bool first_record = true;
+	char *line = NULL;
+	size_t len = 0;
+	ssize_t lineln;
 	while ((lineln = getline(&line, &len, fp)) != -1)
 	{
 		bytecounter += lineln;
 		line_number++;
+
+		if (lineln < job->opt.params.csv_fields)
+		{
+			log_debug("%s: Line %d -- Skipped, the line %s is too short (size %d)\n", job->csv_path, line_number, line, lineln);
+			skipped++;
+			continue;			
+		}
 		//skip lines starting with non alphanumeric char
 		if (!isalnum(line[0]))
 		{
+			log_debug("%s: Line %d -- Skipped, Non alphanumeric char %d on line %s\n", job->csv_path, line_number, line[0], line);
 			skipped++;
 			continue;			
 		}
 		//skip keys with the incorrect lenght.
-		int key_len = strchr(line, ',') - line;
+		char * first_comma = strchr(line, ',');
+		if (!first_comma)
+		{
+			log_debug("%s: Line %d -- Skipped, wrong csv format on line %s .\n", job->csv_path, line_number, line);
+			skipped++;
+			continue;
+		}
+		int key_len = first_comma - line;
 		if (key_len != MD5_LEN_HEX && key_len != MD5_LEN_HEX - 2)
 		{
 			log_debug("%s: Line %d -- Skipped, %d Incorrect key lenght.\n", job->csv_path, line_number, key_len);
@@ -665,6 +679,7 @@ int ldb_import_csv(ldb_importation_config_t * job)
 
 		if (!data)
 		{
+			log_debug("%s: Line %d -- Skipped, data is missed %d.\n", job->csv_path, line_number);
 			skipped++;
 			continue;
 		}
@@ -688,7 +703,7 @@ int ldb_import_csv(ldb_importation_config_t * job)
 				data = field_n(3, line);
 				if (!data)
 				{
-					log_debug("%s: Error in line: %d -- Skipped\n", job->csv_path, line_number);
+					log_debug("%s: Error in line: %d, data is missing -- %s Skipped\n", job->csv_path, line_number, line);
 					skipped++;
 				}
 			}
@@ -767,7 +782,10 @@ int ldb_import_csv(ldb_importation_config_t * job)
 						int error = ldb_node_write(oss_bulk, item_sector, item_lastid, item_buf, item_ptr, 0);
 						//abort in case of error
 						if (error < 0)
+						{
+							log_info("failed to process file %s, last line %s\n", job->csv_path, line);
 							return error;
+						}
 					}
 				}
 				/* Open new sector if needed */
@@ -1835,7 +1853,7 @@ bool ldb_import_command(char * dbtable, char * path, char * config)
 	{
 		ldb_importation_config_parse(&user_opt, config);
 		if (user_opt.params.verbose > 0)
-			logger_set_level(LOG_INFO);
+			logger_set_level(user_opt.params.verbose);
 	}
 
 	if (!ldb_database_exists(job.dbname))

src/ldb.c

@@ -48,6 +48,7 @@ char ldb_root[] = "/var/lib/ldb";
 char ldb_lock_path[] = "/dev/shm/ldb.lock";
 int ldb_cmp_width = 0;
 
+bool ldb_read_failure = false;
 /**
  * @brief Display LDB error and exit program
  * 

src/ldb.h

@@ -27,14 +27,16 @@
 #include "./ldb/types.h"
 #include "./ldb/mz.h"
 
-#define LDB_VERSION "4.1.1"
+#define LDB_VERSION "4.1.2"
 
 #define LDB_TABLE_DEFINITION_UNDEFINED -1
 #define LDB_TABLE_DEFINITION_STANDARD 0
 #define LDB_TABLE_DEFINITION_ENCRYPTED 1
 #define LDB_TABLE_DEFINITION_COMPRESSED 4
 #define LDB_TABLE_DEFINITION_MZ 2
 
+extern bool ldb_read_failure;
+
 bool ldb_file_exists(char *path);
 bool ldb_dir_exists(char *path);
 bool ldb_locked();

src/node.c

@@ -76,7 +76,7 @@ void ldb_load_node(struct ldb_recordset *rs)
 
 	/* Read node */
 	rs->node = malloc(rs->node_ln + 1);
-	if (!fread(rs->node, 1, rs->node_ln, rs->sector)) printf("Warning: cannot load node\n");
+	if (!fread(rs->node, 1, rs->node_ln, rs->sector)) log_info("Warning: cannot load node\n");
 
 	/* Terminate with a chr(0) */
 	rs->node[rs->node_ln] = 0;
@@ -109,11 +109,11 @@ int ldb_node_write (struct ldb_table table, FILE *ldb_sector, uint8_t *key, uint
 	/* Obtain the pointer to the last node of the list */
 	uint64_t list = ldb_list_pointer(ldb_sector, key);
 
-	if (list > 0 && list < LDB_MAP_SIZE) {
-		printf("\nFatal data corruption on list %lu for key %02x%02x%02x%02x\n", list, key[0], key[1], key[2], key[3]);
-		fprintf(stdout, "E057 Map location %08lx\n", ldb_map_pointer_pos(key));
+	if ((list > 0 && list < LDB_MAP_SIZE) || ldb_read_failure) {
+		log_info("\nFatal data corruption on list %lu for key %02x%02x%02x%02x from table %s/%s\n", list, key[0], key[1], key[2], key[3], table.db, table.table);
+		log_info("E057 Map location %08lx\n", ldb_map_pointer_pos(key));
+		ldb_read_failure = false;
 		return LDB_ERROR_MAP_CORRUPTED;
-		//exit(EXIT_FAILURE);
 	}
 
 	/* Seek end of file, and save the pointer to the new node */
@@ -215,7 +215,10 @@ uint64_t ldb_node_read(uint8_t *sector, struct ldb_table table, FILE *ldb_sector
 			ptr = ldb_list_pointer(ldb_sector, key);
 
 		/* If pointer is zero, then there are no records for the key */
-		if (ptr == 0) { return 0; }
+		if (ptr == 0) 
+		{ 
+			return 0; 
+		}
 
 		/* If there is a list, we skip the first bytes (LN: last node pointer) to move into the first node */
 		ptr += LDB_PTR_LN;
@@ -224,12 +227,17 @@ uint64_t ldb_node_read(uint8_t *sector, struct ldb_table table, FILE *ldb_sector
 	uint8_t *buffer;
 
 	/* Read node information into buffer: NN(5) and TS(2/4) */
-	if (sector) buffer = sector + ptr;
+	if (sector) 
+		buffer = sector + ptr;
 	else
 	{
 		fseeko64(ldb_sector, ptr, SEEK_SET);
 		buffer = calloc(LDB_PTR_LN + table.ts_ln + LDB_KEY_LN, 1);
-		if (!fread(buffer, 1, LDB_PTR_LN + table.ts_ln, ldb_sector)) printf("Warning: cannot read LDB node\n");
+		if (!fread(buffer, 1, LDB_PTR_LN + table.ts_ln, ldb_sector))
+		{
+			log_debug("Warning: cannot read LDB node\n");
+			ldb_read_failure = true;
+		} 
 	}
 
 	/* NN: Obtain the next node */
@@ -261,7 +269,11 @@ uint64_t ldb_node_read(uint8_t *sector, struct ldb_table table, FILE *ldb_sector
 		}
 		else
 		{
-			if (!fread(*out, 1, actual_size, ldb_sector)) printf("Warning: cannot read entire LDB node\n");
+			if (!fread(*out, 1, actual_size, ldb_sector)) 
+			{
+				log_debug("Warning: cannot read entire LDB node\n");
+				ldb_read_failure = true;
+			}
 		}
 		*bytes_read = actual_size;
 
@@ -279,7 +291,7 @@ uint64_t ldb_node_read(uint8_t *sector, struct ldb_table table, FILE *ldb_sector
  * @param table Configuration of a table
  * @param key The key of the table
  */
-void ldb_node_unlink (struct ldb_table table, uint8_t *key)
+void ldb_node_unlink(struct ldb_table table, uint8_t *key)
 {
 
 	uint16_t subkeyln = table.key_ln - LDB_KEY_LN;
@@ -323,7 +335,7 @@ void ldb_node_unlink (struct ldb_table table, uint8_t *key)
 					uint8_t *buffer = malloc(LDB_PTR_LN + table.ts_ln + table.key_ln);
 					if (!fread(buffer, 1, LDB_PTR_LN + table.ts_ln, ldb_sector))
 					{
-						printf("Warning: cannot read LDB node info\n");
+						log_info("Warning: cannot read LDB node info\n");
 						break;
 					}
 
@@ -354,7 +366,7 @@ void ldb_node_unlink (struct ldb_table table, uint8_t *key)
 							/* Read K and GS (2) if needed */
 							if (!fread(buffer, 1, get_bytes, ldb_sector))
 							{
-								printf("Warning: cannot read LDB node info (K/GS)\n");
+								log_info("Warning: cannot read LDB node info (K/GS)\n");
 								break;
 							}
 

src/recordset.c

@@ -29,6 +29,7 @@
   * @see https://github.com/scanoss/ldb/blob/master/src/recordset.c
   */
 #include "ldb.h"
+#include "logger.h"
 
 #ifndef LDB_EXTRACT_DEFINED_EXTERN
 int  ldb_extract_record(char **msg, const uint8_t *data, uint32_t dataset_ptr, int record_size, struct ldb_table table, const uint8_t *key, const uint8_t *subkey)
@@ -81,15 +82,24 @@ uint32_t ldb_fetch_recordset(uint8_t *sector, struct ldb_table table, uint8_t* k
 	{
 		/* Read node */
 		next = ldb_node_read(sector, table, ldb_sector, next, key, &node_size, &node, 0);
-		if (!node_size && !next) break; // reached end of list
+		if (ldb_read_failure)
+		{
+			log_info("Error reading table %s/%s - sector %02x: the file is not available or the node doesn't exist\n", table.db, table.table, *sector);
+			ldb_read_failure = false;
+			next = 0;
+		}
+		if (!node_size && !next) 
+			break; // reached end of list
 
 		/* Pass entire node (fixed record length) to handler */
-		if (table.rec_ln) done = ldb_record_handler(key, NULL, 0 , node, node_size, records++, void_ptr);
+		if (table.rec_ln) 
+			done = ldb_record_handler(key, NULL, 0 , node, node_size, records++, void_ptr);
 
 		/* Extract and pass variable-size records to handler */
 		else
 		{
-			if (!ldb_validate_node(node, node_size, subkey_ln)) continue;
+			if (!ldb_validate_node(node, node_size, subkey_ln)) 
+				continue;
 
 			/* Extract datasets from node */
 			node_ptr = 0;
@@ -106,7 +116,8 @@ uint32_t ldb_fetch_recordset(uint8_t *sector, struct ldb_table table, uint8_t* k
 
 				/* Compare subkey */
 				bool key_matched = true;
-				if (!skip_subkey) if (subkey_ln) key_matched = (memcmp(subkey, key + 4, subkey_ln) == 0);
+				if (!skip_subkey && subkey_ln) 
+					key_matched = (memcmp(subkey, key + 4, subkey_ln) == 0);
 
 				if (key_matched)
 				{

test/source/mined/attribution/10.csv

@@ -0,0 +1,10 @@
+000c8aba74e0a452326ffd3591a8cbd4,10441cffac36ce3d4ca6af938f6bf5f0
+0015b9cd03e8c8f70a30546b31348864,10ffcc4297a5f10a4147d518e84ea272
+0056dc87a7a34955159ef77e16629dba,10e03952c97f225d4229d7524619eb7f
+0063408dca20e06c0828d11829a0021a,10d60a0ddbb944de5b7040dbb419df70
+0064ddefbd4ce7cc4b9b3f7775010e64,10ffcc4297a5f10a4147d518e84ea272
+0072295606049db6a09d00918498db1b,10c25b9e4cf853ef114304fd45ecfab9
+00779085e59795b872db276db574c6a9,10675e3a68c6f89eb0b41569c542ce91
+00837c4575fc64e01a78e5d87f298440,10d1495ba46609e55e586ab71fa81f45
+00838aaf468d27419653f9bb5e5748c7,10a8524df41cb07b69ae049421c6f9d5
+0091692946f94de202322efebfca29db,1059dd60915ecde68f43807e66b5b4af