From 4696d5675f4eceebb2c9983f625d4c46e0efbb9b Mon Sep 17 00:00:00 2001 From: aliaspider Date: Sun, 24 Jan 2016 00:24:56 +0100 Subject: [PATCH 1/2] (libretro-db) cleanups. --- libretro-db/Makefile | 1 - libretro-db/c_converter.c | 93 ++++++++++++++------------------------- 2 files changed, 34 insertions(+), 60 deletions(-) diff --git a/libretro-db/Makefile b/libretro-db/Makefile index 52147089b2..c730abf6d6 100644 --- a/libretro-db/Makefile +++ b/libretro-db/Makefile @@ -33,7 +33,6 @@ PLAIN_CONVERTER_C = \ bintree.c \ query.c \ plain_converter.c \ - $(LIBRETRO_COMMON_DIR)/hash/rhash.c \ $(LIBRETRO_COMMON_DIR)/compat/compat_fnmatch.c \ $(LIBRETRO_COMMON_DIR)/file/retro_file.c \ $(LIBRETRO_COMMON_DIR)/compat/compat_strl.c diff --git a/libretro-db/c_converter.c b/libretro-db/c_converter.c index 71d7dc6273..094fd9cb51 100644 --- a/libretro-db/c_converter.c +++ b/libretro-db/c_converter.c @@ -162,26 +162,6 @@ void dat_converter_list_append(dat_converter_list_t* dst, void* item) dst->count++; } -void dat_converter_token_list_dump(dat_converter_list_t* list) -{ - int i; - - switch (list->type) - { - case DAT_CONVERTER_TOKEN_LIST: - for (i = 0; i < list->count; i++) - printf("\x1B[31m(%6i)@%s:%5i:%-5i\x1B[0m\"%s\"\n", i, - list->values[i].token.fname, - list->values[i].token.line_no, - list->values[i].token.column, - list->values[i].token.label); - break; - default: - return; - } - -} - dat_converter_list_t* dat_converter_lexer(char* src, const char* dat_path) { @@ -240,7 +220,6 @@ dat_converter_list_t* dat_converter_lexer(char* src, const char* dat_path) src++; token.column++; } - *src = '\0'; token.label = NULL; dat_converter_list_append(token_list, &token); @@ -251,14 +230,13 @@ dat_converter_list_t* dat_converter_lexer(char* src, const char* dat_path) dat_converter_list_t* dat_parser_table(dat_converter_list_item_t** start_token) { dat_converter_list_t* parsed_table = dat_converter_list_create(DAT_CONVERTER_MAP_LIST); - const char* key = NULL; - + dat_converter_map_t map = {0}; dat_converter_list_item_t* current = *start_token; while (current->token.label) { - if (!key) + if (!map.key) { if (!strcmp(current->token.label, ")")) { @@ -276,7 +254,7 @@ dat_converter_list_t* dat_parser_table(dat_converter_list_item_t** start_token) } else { - key = current->token.label; + map.key = current->token.label; current++; } } @@ -285,9 +263,7 @@ dat_converter_list_t* dat_parser_table(dat_converter_list_item_t** start_token) if (!strcmp(current->token.label, "(")) { current++; - dat_converter_map_t map; map.type = DAT_CONVERTER_LIST_MAP; - map.key = key; map.value.list = dat_parser_table(¤t); dat_converter_list_append(parsed_table, &map); } @@ -301,14 +277,12 @@ dat_converter_list_t* dat_parser_table(dat_converter_list_item_t** start_token) } else { - dat_converter_map_t map; map.type = DAT_CONVERTER_STRING_MAP; - map.key = key; map.value.string = current->token.label; dat_converter_list_append(parsed_table, &map); current++; } - key = NULL; + map.key = NULL; } } @@ -368,11 +342,14 @@ const char* dat_converter_get_match_key(dat_converter_list_t* list, const char* return NULL; } -dat_converter_list_t* dat_parser(dat_converter_list_t* target, dat_converter_list_t* lexer_list, const char* match_key) +dat_converter_list_t* dat_converter_parser(dat_converter_list_t* target, dat_converter_list_t* lexer_list, const char* match_key) { - const char* key = NULL; bool skip = true; dat_converter_list_item_t* current = lexer_list->values; + dat_converter_map_t map; + + map.key = NULL; + map.type = DAT_CONVERTER_LIST_MAP; if (!target) { @@ -386,11 +363,11 @@ dat_converter_list_t* dat_parser(dat_converter_list_t* target, dat_converter_lis while (current->token.label) { - if (!key) + if (!map.key) { if (!strcmp(current->token.label, "game")) skip = false; - key = current->token.label; + map.key = current->token.label; current++; } else @@ -398,12 +375,9 @@ dat_converter_list_t* dat_parser(dat_converter_list_t* target, dat_converter_lis if (!strcmp(current->token.label, "(")) { current++; - dat_converter_map_t map; - map.type = DAT_CONVERTER_LIST_MAP; - map.key = NULL; map.value.list = dat_parser_table(¤t); if (!skip) - { + { if (match_key) { map.key = dat_converter_get_match_key(map.value.list, match_key); @@ -413,13 +387,16 @@ dat_converter_list_t* dat_parser(dat_converter_list_t* target, dat_converter_lis dat_converter_exit(1); } } + else + map.key = NULL; + dat_converter_list_append(target, &map); skip = true; } else dat_converter_list_free(map.value.list); - key = NULL; + map.key = NULL; } else { @@ -641,7 +618,6 @@ int main(int argc, char** argv) const char* rdb_path; const char* match_key = NULL; RFILE* rdb_file; - int i; if (argc < 2) { @@ -664,33 +640,37 @@ int main(int argc, char** argv) int dat_count = argc; char** dat_buffers = (char**)malloc(dat_count * sizeof(*dat_buffers)); - dat_converter_list_t** dat_lexer_lists = (dat_converter_list_t**)malloc(dat_count * sizeof(*dat_lexer_lists)); + char** dat_buffer = dat_buffers; dat_converter_list_t* dat_parser_list = NULL; - for (i = 0; i < dat_count; i++) + while(argc) { + dat_converter_list_t* dat_lexer_list; size_t dat_file_size; - FILE* dat_file = fopen(argv[i], "r"); + FILE* dat_file = fopen(*argv, "r"); if (!dat_file) { - printf("could not open dat file '%s': %s\n", argv[i], strerror(errno)); + printf("could not open dat file '%s': %s\n", *argv, strerror(errno)); dat_converter_exit(1); } fseek(dat_file, 0, SEEK_END); dat_file_size = ftell(dat_file); fseek(dat_file, 0, SEEK_SET); - dat_buffers[i] = (char*)malloc(dat_file_size + 1); - fread(dat_buffers[i], 1, dat_file_size, dat_file); + *dat_buffer = (char*)malloc(dat_file_size + 1); + fread(*dat_buffer, 1, dat_file_size, dat_file); fclose(dat_file); - dat_buffers[i][dat_file_size] = '\0'; + (*dat_buffer)[dat_file_size] = '\0'; - printf("Parsing dat file '%s'...\n", argv[i]); - dat_lexer_lists[i] = dat_converter_lexer(dat_buffers[i], argv[i]); - dat_parser_list = dat_parser(dat_parser_list, dat_lexer_lists[i], match_key); - // dat_converter_token_list_dump(*dat_lexer_dst); + printf("Parsing dat file '%s'...\n", *argv); + dat_lexer_list = dat_converter_lexer(*dat_buffer, *argv); + dat_parser_list = dat_converter_parser(dat_parser_list, dat_lexer_list, match_key); + dat_converter_list_free(dat_lexer_list); + argc--; + argv++; + dat_buffer++; } rdb_file = retro_fopen(rdb_path, RFILE_MODE_WRITE, -1); @@ -710,16 +690,11 @@ int main(int argc, char** argv) retro_fclose(rdb_file); - for (i = 0; i < dat_count; i++) - { - dat_converter_list_free(dat_lexer_lists[i]); - free(dat_buffers[i]); - } - - free(dat_lexer_lists); - free(dat_buffers); dat_converter_list_free(dat_parser_list); + while(dat_count--) + free(dat_buffers[dat_count]); + free(dat_buffers); return 0; } From 7fb86f47de4cf441b4872982251a15d284187996 Mon Sep 17 00:00:00 2001 From: aliaspider Date: Sun, 24 Jan 2016 03:16:25 +0100 Subject: [PATCH 2/2] (libretro-db) c_converter: implement a binary search tree, improves performance when merging multiple dat files considerably. --- libretro-db/c_converter.c | 81 +++++++++++++++++++++++++++------ libretro-db/c_converter_test.sh | 4 +- 2 files changed, 68 insertions(+), 17 deletions(-) diff --git a/libretro-db/c_converter.c b/libretro-db/c_converter.c index 094fd9cb51..ecb094b38a 100644 --- a/libretro-db/c_converter.c +++ b/libretro-db/c_converter.c @@ -38,9 +38,12 @@ typedef struct const char* fname; } dat_converter_token_t; + typedef struct dat_converter_map_t dat_converter_map_t; typedef struct dat_converter_list_t dat_converter_list_t; typedef union dat_converter_list_item_t dat_converter_list_item_t; +typedef struct dat_converter_search_tree_t dat_converter_search_tree_t; +typedef struct dat_converter_bt_node_t dat_converter_bt_node_t; struct dat_converter_map_t { @@ -57,6 +60,7 @@ struct dat_converter_list_t { dat_converter_list_enum type; dat_converter_list_item_t* values; + dat_converter_bt_node_t* bt_root; int count; int capacity; }; @@ -69,6 +73,20 @@ union dat_converter_list_item_t dat_converter_list_t* list; }; + +struct dat_converter_bt_node_t +{ + int index; + dat_converter_bt_node_t* right; + dat_converter_bt_node_t* left; +}; + +int dat_converter_cmp_func(const void *a, const void *b, void *ctx) +{ + return strcmp(a, b); +} + + dat_converter_list_t* dat_converter_list_create(dat_converter_list_enum type) { dat_converter_list_t* list = malloc(sizeof(*list)); @@ -76,9 +94,19 @@ dat_converter_list_t* dat_converter_list_create(dat_converter_list_enum type) list->count = 0; list->capacity = (1 << 2); list->values = (dat_converter_list_item_t*)malloc(sizeof(*list->values) * list->capacity); + list->bt_root = NULL; return list; } +void dat_converter_bt_node_free(dat_converter_bt_node_t* node) +{ + if(!node) + return; + dat_converter_bt_node_free(node->left); + dat_converter_bt_node_free(node->right); + free(node); +} + void dat_converter_list_free(dat_converter_list_t* list) { if (!list) @@ -95,6 +123,7 @@ void dat_converter_list_free(dat_converter_list_t* list) if (list->values[list->count].map.type == DAT_CONVERTER_LIST_MAP) dat_converter_list_free(list->values[list->count].map.value.list); } + dat_converter_bt_node_free(list->bt_root); break; default: break; @@ -104,6 +133,35 @@ void dat_converter_list_free(dat_converter_list_t* list) free(list); } +dat_converter_bt_node_t* dat_converter_bt_node_insert(dat_converter_list_t* list, dat_converter_bt_node_t** node, dat_converter_map_t* map) +{ + assert(map->key); + assert(list->type == DAT_CONVERTER_MAP_LIST); + + if(!*node) + { + *node = calloc(1, sizeof(dat_converter_bt_node_t)); + return *node; + } + + int diff = strcmp(list->values[(*node)->index].map.key, map->key); + + if(diff < 0) + return dat_converter_bt_node_insert(list, &(*node)->left, map); + else if(diff > 0) + return dat_converter_bt_node_insert(list, &(*node)->right, map); + + /* found match */ + + if(list->values[(*node)->index].map.type == DAT_CONVERTER_LIST_MAP) + dat_converter_list_free(list->values[(*node)->index].map.value.list); + + list->values[(*node)->index].map = *map; + + return NULL; +} + + void dat_converter_list_append(dat_converter_list_t* dst, void* item) { if (dst->count == dst->capacity) @@ -127,26 +185,19 @@ void dat_converter_list_append(dat_converter_list_t* dst, void* item) } case DAT_CONVERTER_MAP_LIST: { - int i; dat_converter_map_t* map = (dat_converter_map_t*) item; if(!map->key) - { - dst->values[dst->count].map = *map; - break; - } - for (i = 0; i < dst->count; i++) - { - if (map->key && dst->values[i].map.key && !strcmp(dst->values[i].map.key, map->key)) - break; - } - if (i == dst->count) dst->values[dst->count].map = *map; else { - if (dst->values[i].map.type == DAT_CONVERTER_LIST_MAP) - dat_converter_list_free(dst->values[i].map.value.list); - dst->values[i].map = *map; - return; + dat_converter_bt_node_t* new_node = dat_converter_bt_node_insert(dst, &dst->bt_root, map); + if(new_node) + { + dst->values[dst->count].map = *map; + new_node->index = dst->count; + } + else + return; } break; } diff --git a/libretro-db/c_converter_test.sh b/libretro-db/c_converter_test.sh index 9243327379..0c6bb646a3 100755 --- a/libretro-db/c_converter_test.sh +++ b/libretro-db/c_converter_test.sh @@ -18,7 +18,7 @@ for dat_file in $DAT_dir/*.dat ; do name=`echo "$name" | sed "s/\.dat//"` ./lua_converter "$lua_RDB_outdir/$name.rdb" dat_converter.lua "$dat_file" done -#./lua_converter "$lua_RDB_outdir/merged.rdb" dat_converter.lua rom.crc $DAT_dir/N*.dat +./lua_converter "$lua_RDB_outdir/merged.rdb" dat_converter.lua rom.sha1 $DAT_dir/N*.dat echo echo "===========================================================" @@ -34,7 +34,7 @@ for dat_file in $DAT_dir/*.dat ; do name=`echo "$name" | sed "s/\.dat//"` ./c_converter "$c_RDB_outdir/$name.rdb" "$dat_file" done -#./c_converter "$c_RDB_outdir/merged.rdb" rom.crc $DAT_dir/N*.dat +./c_converter "$c_RDB_outdir/merged.rdb" rom.sha1 $DAT_dir/N*.dat echo echo "==========================================================="