hardnested: reduce disk space for tables (by > 700MBytes) and other minor changes

- compress tables
- minor changes to progress reporting
- free memory on aborts (@iceman1001)

Author: pwpiwi

.gitattributes

@@ -2,3 +2,4 @@
 # prevent binary files from CRLF handling, diff and merge:
 fpga/fpga.bit -crlf -diff
 *.bin -crlf -diff
+*.z -crlf -diff

.gitignore

@@ -12,10 +12,10 @@
 *.map
 *.bin
 !client/hardnested/*.bin
-!client/hardnested/tables/*.bin
 *.dll
 *.moc.cpp
 *.z
+!client/hardnested/tables/*.z
 usb_cmd.lua
 version.c
 client/ui/ui_overlays.h

client/Makefile

@@ -209,8 +209,8 @@ all-static: LDLIBS:=-static $(LDLIBS)
 all-static: proxmark3 flasher fpga_compress
 
 proxmark3: LDLIBS+=$(LUALIB) $(QTLDLIBS)
-proxmark3: $(OBJDIR)/proxmark3.o $(COREOBJS) $(CMDOBJS) $(QTGUIOBJS) $(MULTIARCHOBJS) lualibs/usb_cmd.lua
-	$(LD) $(LDFLAGS) $(OBJDIR)/proxmark3.o $(COREOBJS) $(CMDOBJS) $(QTGUIOBJS) $(MULTIARCHOBJS) $(LDLIBS) -o $@
+proxmark3: $(OBJDIR)/proxmark3.o $(COREOBJS) $(CMDOBJS) $(QTGUIOBJS) $(MULTIARCHOBJS) $(ZLIBOBJS) lualibs/usb_cmd.lua
+	$(LD) $(LDFLAGS) $(OBJDIR)/proxmark3.o $(COREOBJS) $(CMDOBJS) $(QTGUIOBJS) $(MULTIARCHOBJS) $(ZLIBOBJS) $(LDLIBS) -o $@
 
 flasher: $(OBJDIR)/flash.o $(OBJDIR)/flasher.o $(COREOBJS)
 	$(LD) $(LDFLAGS) $^ $(LDLIBS) -o $@

client/cmdhfmfhard.c

@@ -33,14 +33,15 @@
 #include "parity.h"
 #include "hardnested/hardnested_bruteforce.h"
 #include "hardnested/hardnested_bitarray_core.h"
+#include "zlib.h"
 
 #define NUM_CHECK_BITFLIPS_THREADS		(num_CPUs())
 #define NUM_REDUCTION_WORKING_THREADS	(num_CPUs())
 
 #define IGNORE_BITFLIP_THRESHOLD		0.99	// ignore bitflip arrays which have nearly only valid states
 
 #define STATE_FILES_DIRECTORY			"hardnested/tables/"
-#define STATE_FILE_TEMPLATE				"bitflip_%d_%03" PRIx16 "_states.bin"
+#define STATE_FILE_TEMPLATE				"bitflip_%d_%03" PRIx16 "_states.bin.z"
 
 #define DEBUG_KEY_ELIMINATION
 // #define DEBUG_REDUCTION
@@ -240,12 +241,48 @@ static int compare_count_bitflip_bitarrays(const void *b1, const void *b2)
 }
 
 
+static voidpf inflate_malloc(voidpf opaque, uInt items, uInt size)
+{
+	return malloc(items*size);
+}
+
+
+static void inflate_free(voidpf opaque, voidpf address)
+{
+	free(address);
+}
+
+#define OUTPUT_BUFFER_LEN 80
+#define INPUT_BUFFER_LEN 80
+
+//----------------------------------------------------------------------------
+// Initialize decompression of the respective (HF or LF) FPGA stream 
+//----------------------------------------------------------------------------
+static void init_inflate(z_streamp compressed_stream, uint8_t *input_buffer, uint32_t insize, uint8_t *output_buffer, uint32_t outsize)
+{
+
+	// initialize z_stream structure for inflate:
+	compressed_stream->next_in = input_buffer;
+	compressed_stream->avail_in = insize;
+	compressed_stream->next_out = output_buffer;
+	compressed_stream->avail_out = outsize;
+	compressed_stream->zalloc = &inflate_malloc;
+	compressed_stream->zfree = &inflate_free;
+
+	inflateInit2(compressed_stream, 0);
+	
+}
+
+
 static void init_bitflip_bitarrays(void)
 {
 #if defined (DEBUG_REDUCTION)
 	uint8_t line = 0;
 #endif	
 
+
+	z_stream compressed_stream;
+	
 	char state_files_path[strlen(get_my_executable_directory()) + strlen(STATE_FILES_DIRECTORY) + strlen(STATE_FILE_TEMPLATE) + 1];
 	char state_file_name[strlen(STATE_FILE_TEMPLATE)+1];
 
@@ -262,22 +299,31 @@ static void init_bitflip_bitarrays(void)
 			if (statesfile == NULL) {
 				continue;
 			} else {
-				uint32_t *bitset = (uint32_t *)malloc_bitarray(sizeof(uint32_t) * (1<<19));
-				if (bitset == NULL) {
-					printf("Out of memory error in init_bitflip_statelists(). Aborting...\n");
-					fclose(statesfile);
-					exit(4);
-				}
-				size_t bytesread = fread(bitset, 1, sizeof(uint32_t) * (1<<19), statesfile);
-				if (bytesread != sizeof(uint32_t) * (1<<19)) {
-					printf("File read error with %s. Aborting...", state_file_name);
+				fseek(statesfile, 0, SEEK_END);
+				uint32_t filesize = (uint32_t)ftell(statesfile);
+				rewind(statesfile);
+				uint8_t input_buffer[filesize];
+				size_t bytesread = fread(input_buffer, 1, filesize, statesfile);
+				if (bytesread != filesize) {
+					printf("File read error with %s. Aborting...\n", state_file_name);
 					fclose(statesfile);
-					free_bitarray(bitset);
+					inflateEnd(&compressed_stream);
 					exit(5);
 				}
 				fclose(statesfile);
-				uint32_t count = count_states(bitset);
+				uint32_t count = 0;
+				init_inflate(&compressed_stream, input_buffer, filesize, (uint8_t *)&count, sizeof(count));
+				inflate(&compressed_stream, Z_SYNC_FLUSH);
 				if ((float)count/(1<<24) < IGNORE_BITFLIP_THRESHOLD) {
+					uint32_t *bitset = (uint32_t *)malloc_bitarray(sizeof(uint32_t) * (1<<19));
+					if (bitset == NULL) {
+						printf("Out of memory error in init_bitflip_statelists(). Aborting...\n");
+						inflateEnd(&compressed_stream);
+						exit(4);
+					}
+					compressed_stream.next_out = (uint8_t *)bitset;
+					compressed_stream.avail_out = sizeof(uint32_t) * (1<<19);
+					inflate(&compressed_stream, Z_SYNC_FLUSH);
 					effective_bitflip[odd_even][num_effective_bitflips[odd_even]++] = bitflip;
 					bitflip_bitarrays[odd_even][bitflip] = bitset;
 					count_bitflip_bitarrays[odd_even][bitflip] = count;
@@ -289,9 +335,8 @@ static void init_bitflip_bitarrays(void)
 						line = 0;
 					}
 #endif
-				} else {
-					free_bitarray(bitset);
 				}
+				inflateEnd(&compressed_stream);
 			}
 		}
 		effective_bitflip[odd_even][num_effective_bitflips[odd_even]] = 0x400;	// EndOfList marker
@@ -2549,6 +2594,7 @@ int mfnestedhard(uint8_t blockNo, uint8_t keyType, uint8_t *key, uint8_t trgBloc
 				best_first_bytes[0] = best_first_byte_smallest_bitarray;
 				pre_XOR_nonces();
 				prepare_bf_test_nonces(nonces, best_first_bytes[0]);
+				hardnested_print_progress(num_acquired_nonces, "Starting brute force...", expected_brute_force1, 0);
 				key_found = brute_force();
 				free(candidates->states[ODD_STATE]);
 				free(candidates->states[EVEN_STATE]);
@@ -2568,6 +2614,7 @@ int mfnestedhard(uint8_t blockNo, uint8_t keyType, uint8_t *key, uint8_t trgBloc
 					// printf("Estimated remaining states: %" PRIu64 " (2^%1.1f)\n", nonces[best_first_bytes[0]].sum_a8_guess[j].num_states, log(nonces[best_first_bytes[0]].sum_a8_guess[j].num_states)/log(2.0));
 					generate_candidates(first_byte_Sum, nonces[best_first_bytes[0]].sum_a8_guess[j].sum_a8_idx);
 					// printf("Time for generating key candidates list: %1.0f sec (%1.1f sec CPU)\n", difftime(time(NULL), start_time), (float)(msclock() - start_clock)/1000.0);
+					hardnested_print_progress(num_acquired_nonces, "Starting brute force...", expected_brute_force, 0);
 					key_found = brute_force();
 					free_statelist_cache();
 					free_candidates_memory(candidates);
@@ -2608,6 +2655,12 @@ int mfnestedhard(uint8_t blockNo, uint8_t keyType, uint8_t *key, uint8_t trgBloc
 
 		if (nonce_file_read) {  	// use pre-acquired data from file nonces.bin
 			if (read_nonce_file() != 0) {
+				free_bitflip_bitarrays();
+				free_nonces_memory();
+				free_bitarray(all_bitflips_bitarray[ODD_STATE]);
+				free_bitarray(all_bitflips_bitarray[EVEN_STATE]);
+				free_sum_bitarrays();
+				free_part_sum_bitarrays();
 				return 3;
 			}
 			hardnested_stage = CHECK_1ST_BYTES | CHECK_2ND_BYTES;
@@ -2617,6 +2670,12 @@ int mfnestedhard(uint8_t blockNo, uint8_t keyType, uint8_t *key, uint8_t trgBloc
 		} else {					// acquire nonces.
 			uint16_t is_OK = acquire_nonces(blockNo, keyType, key, trgBlockNo, trgKeyType, nonce_file_write, slow);
 			if (is_OK != 0) {
+				free_bitflip_bitarrays();
+				free_nonces_memory();
+				free_bitarray(all_bitflips_bitarray[ODD_STATE]);
+				free_bitarray(all_bitflips_bitarray[EVEN_STATE]);
+				free_sum_bitarrays();
+				free_part_sum_bitarrays();
 				return is_OK;
 			}
 		}
@@ -2646,10 +2705,11 @@ int mfnestedhard(uint8_t blockNo, uint8_t keyType, uint8_t *key, uint8_t trgBloc
 			for (statelist_t *sl = candidates; sl != NULL; sl = sl->next) {
 				maximum_states += (uint64_t)sl->len[ODD_STATE] * sl->len[EVEN_STATE];
 			}
-			printf("Number of remaining possible keys: %" PRIu64 " (2^%1.1f)\n", maximum_states, log(maximum_states)/log(2.0));
+			// printf("Number of remaining possible keys: %" PRIu64 " (2^%1.1f)\n", maximum_states, log(maximum_states)/log(2.0));
 			best_first_bytes[0] = best_first_byte_smallest_bitarray;
 			pre_XOR_nonces();
 			prepare_bf_test_nonces(nonces, best_first_bytes[0]);
+			hardnested_print_progress(num_acquired_nonces, "Starting brute force...", expected_brute_force1, 0);
 			key_found = brute_force();
 			free(candidates->states[ODD_STATE]);
 			free(candidates->states[EVEN_STATE]);
@@ -2669,6 +2729,7 @@ int mfnestedhard(uint8_t blockNo, uint8_t keyType, uint8_t *key, uint8_t trgBloc
 				// printf("Estimated remaining states: %" PRIu64 " (2^%1.1f)\n", nonces[best_first_bytes[0]].sum_a8_guess[j].num_states, log(nonces[best_first_bytes[0]].sum_a8_guess[j].num_states)/log(2.0));
 				generate_candidates(first_byte_Sum, nonces[best_first_bytes[0]].sum_a8_guess[j].sum_a8_idx);
 				// printf("Time for generating key candidates list: %1.0f sec (%1.1f sec CPU)\n", difftime(time(NULL), start_time), (float)(msclock() - start_clock)/1000.0);
+				hardnested_print_progress(num_acquired_nonces, "Starting brute force...", expected_brute_force, 0);
 				key_found = brute_force();
 				free_statelist_cache();
 				free_candidates_memory(candidates);

client/fpga_compress.c

@@ -38,14 +38,17 @@
 #define COMPRESS_MAX_CHAIN            8192
 
 #define FPGA_INTERLEAVE_SIZE           288  // (the FPGA's internal config frame size is 288 bits. Interleaving with 288 bytes should give best compression)
-#define FPGA_CONFIG_SIZE             42336  // our current fpga_[lh]f.bit files are 42175 bytes. Rounded up to next multiple of FPGA_INTERLEAVE_SIZE
+#define FPGA_CONFIG_SIZE            42336L  // our current fpga_[lh]f.bit files are 42175 bytes. Rounded up to next multiple of FPGA_INTERLEAVE_SIZE
+#define HARDNESTED_TABLE_SIZE		(sizeof(uint32_t) * ((1L<<19)+1))
 
 static void usage(void)
 {
 	fprintf(stderr, "Usage: fpga_compress <infile1> <infile2> ... <infile_n> <outfile>\n");
 	fprintf(stderr, "          Combine n FPGA bitstream files and compress them into one.\n\n");
 	fprintf(stderr, "       fpga_compress -d <infile> <outfile>");
 	fprintf(stderr, "          Decompress <infile>. Write result to <outfile>");
+	fprintf(stderr, "       fpga_compress -t <infile> <outfile>");
+	fprintf(stderr, "          Compress hardnested table <infile>. Write result to <outfile>");
 }
 
 
@@ -73,22 +76,29 @@ static bool all_feof(FILE *infile[], uint8_t num_infiles)
 }
 
 
-int zlib_compress(FILE *infile[], uint8_t num_infiles, FILE *outfile)
+int zlib_compress(FILE *infile[], uint8_t num_infiles, FILE *outfile, bool hardnested_mode)
 {
 	uint8_t *fpga_config;
 	uint32_t i;
 	int32_t ret;
 	uint8_t c;
 	z_stream compressed_fpga_stream;
 
-	fpga_config = malloc(num_infiles * FPGA_CONFIG_SIZE);
-	
+	if (hardnested_mode) {
+		fpga_config = malloc(num_infiles * HARDNESTED_TABLE_SIZE);
+	} else {
+		fpga_config = malloc(num_infiles * FPGA_CONFIG_SIZE);
+	}		
 	// read the input files. Interleave them into fpga_config[]
 	i = 0;
 	do {
 
-		if (i >= num_infiles * FPGA_CONFIG_SIZE) {
-			fprintf(stderr, "Input files too big (total > %u bytes). These are probably not PM3 FPGA config files.\n", num_infiles*FPGA_CONFIG_SIZE);
+		if (i >= num_infiles * (hardnested_mode?HARDNESTED_TABLE_SIZE:FPGA_CONFIG_SIZE)) {
+			if (hardnested_mode) {
+				fprintf(stderr, "Input file too big (> %lu bytes). This is probably not a hardnested bitflip state table.\n", HARDNESTED_TABLE_SIZE);
+			} else {
+				fprintf(stderr, "Input files too big (total > %lu bytes). These are probably not PM3 FPGA config files.\n", num_infiles*FPGA_CONFIG_SIZE);
+			}
 			for(uint16_t j = 0; j < num_infiles; j++) {
 				fclose(infile[j]);
 			}
@@ -253,6 +263,7 @@ int main(int argc, char **argv)
 	}
 
 	if (!strcmp(argv[1], "-d")) { // Decompress
+
 		infiles = calloc(1, sizeof(FILE*));
 		if (argc != 4) {
 			usage();
@@ -272,11 +283,23 @@ int main(int argc, char **argv)
 
 	} else { // Compress
 
-		infiles = calloc(argc-2, sizeof(FILE*));
-		for (uint16_t i = 0; i < argc-2; i++) { 
-			infiles[i] = fopen(argv[i+1], "rb");
+		bool hardnested_mode = false;
+		int num_input_files = 0;
+		if (!strcmp(argv[1], "-t")) { // hardnested table
+			if (argc != 4) {
+				usage();
+				return(EXIT_FAILURE);
+			}
+			hardnested_mode = true;
+			num_input_files = 1;
+		} else {
+			num_input_files = argc-2;
+		}
+		infiles = calloc(num_input_files, sizeof(FILE*));
+		for (uint16_t i = 0; i < num_input_files; i++) { 
+			infiles[i] = fopen(argv[i+hardnested_mode?2:1], "rb");
 			if (infiles[i] == NULL) {
-				fprintf(stderr, "Error. Cannot open input file %s", argv[i+1]);
+				fprintf(stderr, "Error. Cannot open input file %s", argv[i+hardnested_mode?2:1]);
 				return(EXIT_FAILURE);
 			}
 		}
@@ -285,6 +308,6 @@ int main(int argc, char **argv)
 			fprintf(stderr, "Error. Cannot open output file %s", argv[argc-1]);
 			return(EXIT_FAILURE);
 		}
-		return zlib_compress(infiles, argc-2, outfile);
+		return zlib_compress(infiles, num_input_files, outfile, hardnested_mode);
 	}
 }

client/hardnested/hardnested_tables.c

@@ -176,11 +176,12 @@ static inline uint32_t count_states(uint32_t *bitset)
 }
 
 
-static void write_bitflips_file(odd_even_t odd_even, uint16_t bitflip, int sum_a0, uint32_t *bitset)
+static void write_bitflips_file(odd_even_t odd_even, uint16_t bitflip, int sum_a0, uint32_t *bitset, uint32_t count)
 {
 	char filename[80];
 	sprintf(filename, "bitflip_%d_%03" PRIx16 "_sum%d_states.bin", odd_even, bitflip, sum_a0);
 	FILE *outfile = fopen(filename, "wb");
+	fwrite(&count, 1, sizeof(count), outfile);
 	fwrite(bitset, 1, sizeof(uint32_t)*(1<<19), outfile);
 	fclose(outfile);
 }
@@ -369,7 +370,7 @@ static void precalculate_bit0_bitflip_bitarrays(uint8_t const bitflip, uint16_t
 				bitflip, (1<<24) - count[odd_even],
 				(float)((1<<24) - count[odd_even]) / (1<<24) * 100.0);
 			#ifndef TEST_RUN
-			write_bitflips_file(odd_even, bitflip, sum_a0, test_bitarray[odd_even]);
+			write_bitflips_file(odd_even, bitflip, sum_a0, test_bitarray[odd_even], count[odd_even]);
 			#endif
 		} else {
 			printf("All %s states for bitflip property %03x are possible. No file written.\n", odd_even==EVEN_STATE?"even":"odd", bitflip);
@@ -396,7 +397,7 @@ static void precalculate_bit0_bitflip_bitarrays(uint8_t const bitflip, uint16_t
 					bitflip | BITFLIP_2ND_BYTE, (1<<24) - count[odd_even],
 					(float)((1<<24) - count[odd_even]) / (1<<24) * 100.0);
 				#ifndef TEST_RUN
-				write_bitflips_file(odd_even, bitflip | BITFLIP_2ND_BYTE, sum_a0, test_bitarray_2nd);
+				write_bitflips_file(odd_even, bitflip | BITFLIP_2ND_BYTE, sum_a0, test_bitarray_2nd, count[odd_even]);
 				#endif
 			} else {
 				printf("All %s states for bitflip property %03x are possible. No file written.\n", odd_even==EVEN_STATE?"even":"odd", bitflip | BITFLIP_2ND_BYTE);
@@ -481,7 +482,7 @@ static void precalculate_bit0_bitflip_bitarrays(uint8_t const bitflip, uint16_t
 				bitflip|0x100, (1<<24) - count[odd_even],
 				(float)((1<<24) - count[odd_even]) / (1<<24) * 100.0);
 			#ifndef TEST_RUN
-			write_bitflips_file(odd_even, bitflip|0x100, sum_a0, test_not_bitarray[odd_even]);
+			write_bitflips_file(odd_even, bitflip|0x100, sum_a0, test_not_bitarray[odd_even], count[odd_even]);
 			#endif
 		} else {
 			printf("All %s states for bitflip property %03x are possible. No file written.\n", odd_even==EVEN_STATE?"even":"odd", bitflip|0x100);
@@ -508,7 +509,7 @@ static void precalculate_bit0_bitflip_bitarrays(uint8_t const bitflip, uint16_t
 					bitflip | 0x100| BITFLIP_2ND_BYTE, (1<<24) - count[odd_even],
 					(float)((1<<24) - count[odd_even]) / (1<<24) * 100.0);
 				#ifndef TEST_RUN
-				write_bitflips_file(odd_even, bitflip | 0x100 | BITFLIP_2ND_BYTE, sum_a0, test_bitarray_2nd);
+				write_bitflips_file(odd_even, bitflip | 0x100 | BITFLIP_2ND_BYTE, sum_a0, test_bitarray_2nd, count[odd_even]);
 				#endif
 			} else {
 				printf("All %s states for bitflip property %03x are possible. No file written.\n", odd_even==EVEN_STATE?"even":"odd", bitflip | 0x100 | BITFLIP_2ND_BYTE);