From dc02b0e05e9cace31fa34cf8d3a49636e83c08e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Friedel=20Sch=C3=B6n?= Date: Fri, 12 Jan 2024 16:51:36 +0100 Subject: [PATCH 01/12] lib,bin: implementing xbps-locate and adding files.plist to *-repodata --- TODO | 3 + bin/Makefile | 1 + bin/xbps-locate/Makefile | 7 + bin/xbps-locate/main.c | 256 ++++++++++++++++++++++++++++++++++ bin/xbps-rindex/defs.h | 2 +- bin/xbps-rindex/index-add.c | 62 +++++++- bin/xbps-rindex/index-clean.c | 2 +- bin/xbps-rindex/repoflush.c | 33 +++-- bin/xbps-rindex/sign.c | 2 +- include/xbps.h.in | 14 +- lib/repo.c | 37 ++++- 11 files changed, 392 insertions(+), 27 deletions(-) create mode 100644 bin/xbps-locate/Makefile create mode 100644 bin/xbps-locate/main.c diff --git a/TODO b/TODO index eb95ee6e3..405b5bf4b 100644 --- a/TODO +++ b/TODO @@ -21,4 +21,7 @@ xbps-fetch: xbps-digest: - blake2b support +xbps-rindex: + - clean should also clean files.plist entries + Issues listed at https://github.com/void-linux/xbps/issues diff --git a/bin/Makefile b/bin/Makefile index c0586aabb..417caf345 100644 --- a/bin/Makefile +++ b/bin/Makefile @@ -14,6 +14,7 @@ SUBDIRS += xbps-checkvers SUBDIRS += xbps-fbulk SUBDIRS += xbps-digest SUBDIRS += xbps-fetch +SUBDIRS += xbps-locate ifeq (${XBPS_OS},linux) SUBDIRS += xbps-uchroot diff --git a/bin/xbps-locate/Makefile b/bin/xbps-locate/Makefile new file mode 100644 index 000000000..067a72539 --- /dev/null +++ b/bin/xbps-locate/Makefile @@ -0,0 +1,7 @@ +TOPDIR = ../.. +-include $(TOPDIR)/config.mk + +BIN = xbps-locate +OBJS = main.o + +include $(TOPDIR)/mk/prog.mk diff --git a/bin/xbps-locate/main.c b/bin/xbps-locate/main.c new file mode 100644 index 000000000..57f98482d --- /dev/null +++ b/bin/xbps-locate/main.c @@ -0,0 +1,256 @@ +/*- + * Copyright (c) 2008-2015 Juan Romero Pardines. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + + +struct locate { + const char* expr; + bool case_ignore; + regex_t regex; +}; + +static void __attribute__((noreturn)) +usage(bool fail) +{ + fprintf(stdout, "Usage: xbps-locate [OPTIONS] file-pattern...\n" + "\n" + "OPTIONS\n" + " -C, --config Path to confdir (xbps.d)\n" + " -c, --cachedir Path to cachedir\n" + " -d, --debug Debug mode shown to stderr\n" + " -h, --help Show usage\n" + " -e, --regex Use extended regular expression pattern\n" + " -i, --ignore-conf-repos Ignore repositories defined in xbps.d\n" + " -I, --ignore-case Match case insensitive\n" + " -M, --memory-sync Remote repository data is fetched and stored\n" + " in memory, ignoring on-disk repodata archives\n" + " -R, --repository Add repository to the top of the list\n" + " This option can be specified multiple times\n" + " -r, --rootdir Full path to rootdir\n" + " -V, --version Show XBPS version\n" + " -v, --verbose Verbose messages\n"); + + exit(fail ? EXIT_FAILURE : EXIT_SUCCESS); +} + +#define LOWERWHEN(ignorecase, chr) (!(ignorecase) ? (chr) : tolower(chr)) + +static bool strcontains(const char* haystack, const char* needle, bool ignorecase) { + const char *a, *b; + + b = needle; + for ( ; *haystack != 0; haystack++) { + if (*haystack != *b) { + continue; + } + a = haystack; + do { + if (*b == 0) + return true; + } while (LOWERWHEN(ignorecase, *a++) == LOWERWHEN(ignorecase, *b++)); + b = needle; + } + return false; +} + +static int repo_search_files(struct xbps_repo* repo, void* locate_ptr, bool* done) { + struct locate *locate = locate_ptr; + const char *pkgname, *file; + xbps_object_iterator_t iter; + xbps_object_t pkgkey; + xbps_array_t pkgfiles; + + (void) done; + + if (repo->files == NULL) { + xbps_warn_printf("repository %s has no files-entry, skipping.\n", repo->uri); + return 1; + } + + iter = xbps_dictionary_iterator(repo->files); + + while ((pkgkey = xbps_object_iterator_next(iter)) != NULL) { + pkgname = xbps_dictionary_keysym_cstring_nocopy(pkgkey); + pkgfiles = xbps_dictionary_get_keysym(repo->files, pkgkey); + + for (unsigned int i = 0; i < xbps_array_count(pkgfiles); i++) { + if (!xbps_array_get_cstring_nocopy(pkgfiles, i, &file)) + continue; + if (locate->expr != NULL) { + if (strcontains(file, locate->expr, locate->case_ignore)) + printf("%s: %s\n", pkgname, file); + } else { + if (!regexec(&locate->regex, file, 0, NULL, 0)) + printf("%s: %s\n", pkgname, file); + } + } + } + + xbps_object_iterator_release(iter); + + return 1; +} + +int +main(int argc, char **argv) +{ + const char *shortopts = "C:c:dehiIMr:R:r:vV"; + const struct option longopts[] = { + { "config", required_argument, NULL, 'C' }, + { "cachedir", required_argument, NULL, 'c' }, + { "debug", no_argument, NULL, 'd' }, + { "regex", no_argument, NULL, 'e' }, + { "help", no_argument, NULL, 'h' }, + { "ignore-conf-repos", no_argument, NULL, 'i' }, + { "ignore-case", no_argument, NULL, 'I' }, + { "memory-sync", no_argument, NULL, 'M' }, + { "regex", no_argument, NULL, 'r' }, + { "repository", required_argument, NULL, 'R' }, + { "rootdir", required_argument, NULL, 'r' }, + { "verbose", no_argument, NULL, 'v' }, + { "version", no_argument, NULL, 'V' }, + { NULL, 0, NULL, 0 }, + }; + + struct xbps_handle xh; + struct locate locate; + const char *rootdir, *cachedir, *confdir; + int c, flags, rv, regflags; + bool regex; + + rootdir = cachedir = confdir = NULL; + flags = rv = c = 0; + regex = false; + regflags = REG_EXTENDED | REG_NOSUB; + + locate.expr = NULL; + locate.case_ignore = false; + + memset(&xh, 0, sizeof(xh)); + + while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) { + switch (c) { + case 'C': + confdir = optarg; + break; + case 'c': + cachedir = optarg; + break; + case 'd': + flags |= XBPS_FLAG_DEBUG; + break; + case 'h': + usage(false); + /* NOTREACHED */ + case 'i': + flags |= XBPS_FLAG_IGNORE_CONF_REPOS; + break; + case 'I': + locate.case_ignore = true; + regflags |= REG_ICASE; + break; + case 'M': + flags |= XBPS_FLAG_REPOS_MEMSYNC; + break; + case 'R': + xbps_repo_store(&xh, optarg); + break; + case 'r': + rootdir = optarg; + break; + case 'v': + flags |= XBPS_FLAG_VERBOSE; + break; + case 'V': + printf("%s\n", XBPS_RELVER); + exit(EXIT_SUCCESS); + case 'e': + regex = true; + break; + case '?': + default: + usage(true); + /* NOTREACHED */ + } + } + argc -= optind; + argv += optind; + + if (!argc) { + usage(true); + /* NOTREACHED */ + } + + if (regex) { + if (regcomp(&locate.regex, *(argv++), regflags) != 0) { + xbps_error_printf("invalid regular expression\n"); + exit(1); + } + } else { + locate.expr = *(argv++); + } + argc--; + + if (argc) { + /* trailing parameters */ + usage(true); + /* NOTREACHED */ + } + /* + * Initialize libxbps. + */ + if (rootdir) + xbps_strlcpy(xh.rootdir, rootdir, sizeof(xh.rootdir)); + if (cachedir) + xbps_strlcpy(xh.cachedir, cachedir, sizeof(xh.cachedir)); + if (confdir) + xbps_strlcpy(xh.confdir, confdir, sizeof(xh.confdir)); + + xh.flags = flags; + + if ((rv = xbps_init(&xh)) != 0) { + xbps_error_printf("Failed to initialize libxbps: %s\n", + strerror(rv)); + exit(EXIT_FAILURE); + } + + xbps_rpool_foreach(&xh, repo_search_files, &locate); + + if (regex) + regfree(&locate.regex); + + xbps_end(&xh); + exit(rv); +} diff --git a/bin/xbps-rindex/defs.h b/bin/xbps-rindex/defs.h index d99b59f38..2c70fcf5d 100644 --- a/bin/xbps-rindex/defs.h +++ b/bin/xbps-rindex/defs.h @@ -46,6 +46,6 @@ int sign_pkgs(struct xbps_handle *, int, int, char **, const char *, bool); /* From repoflush.c */ bool repodata_flush(struct xbps_handle *, const char *, const char *, - xbps_dictionary_t, xbps_dictionary_t, const char *); + xbps_dictionary_t, xbps_dictionary_t, xbps_dictionary_t, const char *); #endif /* !_XBPS_RINDEX_DEFS_H_ */ diff --git a/bin/xbps-rindex/index-add.c b/bin/xbps-rindex/index-add.c index 00b3132c9..ecb306975 100644 --- a/bin/xbps-rindex/index-add.c +++ b/bin/xbps-rindex/index-add.c @@ -41,7 +41,7 @@ static bool repodata_commit(struct xbps_handle *xhp, const char *repodir, - xbps_dictionary_t idx, xbps_dictionary_t meta, xbps_dictionary_t stage, + xbps_dictionary_t idx, xbps_dictionary_t meta,xbps_dictionary_t files, xbps_dictionary_t stage, const char *compression) { xbps_object_iterator_t iter; @@ -174,7 +174,7 @@ repodata_commit(struct xbps_handle *xhp, const char *repodir, printf("stage: added `%s' (%s)\n", pkgver, arch); } xbps_object_iterator_release(iter); - rv = repodata_flush(xhp, repodir, "stagedata", stage, NULL, compression); + rv = repodata_flush(xhp, repodir, "stagedata", stage, NULL, files, compression); } else { char *stagefile; @@ -192,17 +192,52 @@ repodata_commit(struct xbps_handle *xhp, const char *repodir, stagefile = xbps_repo_path_with_name(xhp, repodir, "stagedata"); unlink(stagefile); free(stagefile); - rv = repodata_flush(xhp, repodir, "repodata", idx, meta, compression); + rv = repodata_flush(xhp, repodir, "repodata", idx, meta, files, compression); } xbps_object_release(usedshlibs); xbps_object_release(oldshlibs); return rv; } +static xbps_array_t create_file_list(const char* pkg, xbps_dictionary_t filepkg) { + xbps_array_t file_list = xbps_array_create(); + xbps_array_t files; + const char* path; + + if ((files = xbps_dictionary_get(filepkg, "dirs")) == NULL) { + xbps_error_printf("index: failed to get `dirs' from files.plist in `%s'\n", pkg); + xbps_object_release(file_list); + return NULL; + } + + for (unsigned int i = 0; i < xbps_array_count(files); i++) { + if (!xbps_dictionary_get_cstring_nocopy(xbps_array_get(files, i), "file", &path)) + continue; + + xbps_array_add_cstring(file_list, path); + } + + if ((files = xbps_dictionary_get(filepkg, "files")) == NULL) { + xbps_error_printf("index: failed to get `files' from files.plist in `%s'\n", pkg); + xbps_object_release(file_list); + return NULL; + } + + for (unsigned int i = 0; i < xbps_array_count(files); i++) { + if (!xbps_dictionary_get_cstring_nocopy(xbps_array_get(files, i), "file", &path)) + continue; + + xbps_array_add_cstring(file_list, path); + } + + return file_list; +} + int index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force, const char *compression) { - xbps_dictionary_t idx, idxmeta, idxstage, binpkgd, curpkgd; + xbps_dictionary_t idx, idxmeta, idxstage, files, binpkgd, curpkgd, filespkgd; + xbps_array_t file_list; struct xbps_repo *repo = NULL, *stage = NULL; struct stat st; char *tmprepodir = NULL, *repodir = NULL, *rlockfname = NULL; @@ -232,9 +267,11 @@ index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force if (repo) { idx = xbps_dictionary_copy_mutable(repo->idx); idxmeta = xbps_dictionary_copy_mutable(repo->idxmeta); + files = xbps_dictionary_copy_mutable(repo->files); } else { idx = xbps_dictionary_create(); idxmeta = NULL; + files = xbps_dictionary_create(); } stage = xbps_repo_stage_open(xhp, repodir); if (stage == NULL && errno != ENOENT) { @@ -268,11 +305,18 @@ index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force "`%s', skipping!\n", XBPS_PKGPROPS, pkg); continue; } + filespkgd = xbps_archive_fetch_plist(pkg, "/files.plist"); + if (filespkgd == NULL) { + xbps_warn_printf("index: failed to read files.plist metadata for " + "`%s'!\n", pkg); + } xbps_dictionary_get_cstring_nocopy(binpkgd, "architecture", &arch); xbps_dictionary_get_cstring(binpkgd, "pkgver", &pkgver); if (!xbps_pkg_arch_match(xhp, arch, NULL)) { fprintf(stderr, "index: ignoring %s, unmatched arch (%s)\n", pkgver, arch); xbps_object_release(binpkgd); + if (filespkgd) + xbps_object_release(filespkgd); free(pkgver); continue; } @@ -363,6 +407,14 @@ index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force xbps_dictionary_remove(binpkgd, "version"); xbps_dictionary_remove(binpkgd, "packaged-with"); + if ((file_list = create_file_list(pkg, filespkgd)) == NULL) { + xbps_warn_printf("index: failed to create filelist for `%s', skipping.\n", pkg); + } else { + xbps_dictionary_set(files, pkgname, file_list); + } + xbps_object_release(filespkgd); + xbps_object_release(file_list); + /* * Add new pkg dictionary into the stage index */ @@ -378,7 +430,7 @@ index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force /* * Generate repository data files. */ - if (!repodata_commit(xhp, repodir, idx, idxmeta, idxstage, compression)) { + if (!repodata_commit(xhp, repodir, idx, idxmeta, files, idxstage, compression)) { xbps_error_printf("%s: failed to write repodata: %s\n", _XBPS_RINDEX, strerror(errno)); goto out; diff --git a/bin/xbps-rindex/index-clean.c b/bin/xbps-rindex/index-clean.c index 2226e19f7..d415cd887 100644 --- a/bin/xbps-rindex/index-clean.c +++ b/bin/xbps-rindex/index-clean.c @@ -116,7 +116,7 @@ cleanup_repo(struct xbps_handle *xhp, const char *repodir, struct xbps_repo *rep free(stagefile); } if (!xbps_dictionary_equals(dest, repo->idx)) { - if (!repodata_flush(xhp, repodir, reponame, dest, repo->idxmeta, compression)) { + if (!repodata_flush(xhp, repodir, reponame, dest, repo->idxmeta, repo->files, compression)) { rv = errno; xbps_error_printf("failed to write repodata: %s\n", strerror(errno)); diff --git a/bin/xbps-rindex/repoflush.c b/bin/xbps-rindex/repoflush.c index 010c25919..e43f5739d 100644 --- a/bin/xbps-rindex/repoflush.c +++ b/bin/xbps-rindex/repoflush.c @@ -41,10 +41,11 @@ #include #include "defs.h" +#include "xbps/xbps_dictionary.h" bool repodata_flush(struct xbps_handle *xhp, const char *repodir, - const char *reponame, xbps_dictionary_t idx, xbps_dictionary_t meta, + const char *reponame, xbps_dictionary_t idx, xbps_dictionary_t meta, xbps_dictionary_t files, const char *compression) { struct archive *ar; @@ -106,20 +107,28 @@ repodata_flush(struct xbps_handle *xhp, const char *repodir, if (rv != 0) return false; - /* XBPS_REPOIDX_META */ - if (meta == NULL) { - /* fake entry */ - buf = strdup("DEADBEEF"); + if (meta != NULL) { /* XBPS_REPOIDX_META */ + buf = xbps_dictionary_externalize(meta); if (buf == NULL) return false; - } else { - buf = xbps_dictionary_externalize(meta); + rv = xbps_archive_append_buf(ar, buf, strlen(buf), + XBPS_REPOIDX_META, 0644, "root", "root"); + free(buf); + if (rv != 0) + return false; + } + + if (files != NULL) { + /* XBPS_REPOIDX_META */ + buf = xbps_dictionary_externalize(files); + if (buf == NULL) + return false; + rv = xbps_archive_append_buf(ar, buf, strlen(buf), + XBPS_REPO_FILES, 0644, "root", "root"); + free(buf); + if (rv != 0) + return false; } - rv = xbps_archive_append_buf(ar, buf, strlen(buf), - XBPS_REPOIDX_META, 0644, "root", "root"); - free(buf); - if (rv != 0) - return false; /* Write data to tempfile and rename */ if (archive_write_close(ar) != ARCHIVE_OK) diff --git a/bin/xbps-rindex/sign.c b/bin/xbps-rindex/sign.c index 7e6cd2ce2..7179ba1bd 100644 --- a/bin/xbps-rindex/sign.c +++ b/bin/xbps-rindex/sign.c @@ -222,7 +222,7 @@ sign_repo(struct xbps_handle *xhp, const char *repodir, _XBPS_RINDEX, strerror(errno)); goto out; } - flush_failed = repodata_flush(xhp, repodir, "repodata", repo->idx, meta, compression); + flush_failed = repodata_flush(xhp, repodir, "repodata", repo->idx, meta, repo->files, compression); xbps_repo_unlock(rlockfd, rlockfname); if (!flush_failed) { xbps_error_printf("failed to write repodata: %s\n", strerror(errno)); diff --git a/include/xbps.h.in b/include/xbps.h.in index 5ccd51e94..cb992ab7f 100644 --- a/include/xbps.h.in +++ b/include/xbps.h.in @@ -121,6 +121,12 @@ */ #define XBPS_REPOIDX_META "index-meta.plist" +/** + * @def XBPS_REPO_FILES + * Filename for the repository files property list. + */ +#define XBPS_REPO_FILES "files.plist" + /** * @def XBPS_FLAG_VERBOSE * Verbose flag that can be used in the function callbacks to alter @@ -1431,7 +1437,13 @@ struct xbps_repo { * * Proplib dictionary associated with the repository index-meta. */ - xbps_dictionary_t idxmeta; + xbps_dictionary_t idxmeta; + /** + * @var files + * + * Proplib dictionary associated with the repository files. + */ + xbps_dictionary_t files; /** * @var uri * diff --git a/lib/repo.c b/lib/repo.c index 8d2aa6bd1..182075692 100644 --- a/lib/repo.c +++ b/lib/repo.c @@ -58,14 +58,14 @@ xbps_repo_path_with_name(struct xbps_handle *xhp, const char *url, const char *n { assert(xhp); assert(url); - assert(strcmp(name, "repodata") == 0 || strcmp(name, "stagedata") == 0); + assert(strcmp(name, "repodata") == 0 || strcmp(name, "stagedata") == 0 || strcmp(name, "files") == 0); return xbps_xasprintf("%s/%s-%s", url, xhp->target_arch ? xhp->target_arch : xhp->native_arch, name); } static xbps_dictionary_t -repo_get_dict(struct xbps_repo *repo) +repo_get_dict(struct xbps_repo *repo, const char** filename) { struct archive_entry *entry; int rv; @@ -79,6 +79,7 @@ repo_get_dict(struct xbps_repo *repo) archive_error_string(repo->ar)); return NULL; } + *filename = archive_entry_pathname(entry); return xbps_archive_get_dictionary(repo->ar, entry); } @@ -136,6 +137,8 @@ static bool repo_open_local(struct xbps_repo *repo, const char *repofile) { struct stat st; + const char* filename; + xbps_dictionary_t dict; if (fstat(repo->fd, &st) == -1) { xbps_dbg_printf("[repo] `%s' fstat repodata %s\n", @@ -157,19 +160,41 @@ repo_open_local(struct xbps_repo *repo, const char *repofile) repofile, archive_error_string(repo->ar)); return false; } - if ((repo->idx = repo_get_dict(repo)) == NULL) { - xbps_dbg_printf("[repo] `%s' failed to internalize " - " index on archive, removing file.\n", repofile); + + repo->idx = NULL; + repo->idxmeta = NULL; + repo->files = NULL; + while ((dict=repo_get_dict(repo, &filename)) != NULL) { + if (strcmp(filename, "index.plist") == 0) { + repo->idx = dict; + } else if (strcmp(filename, "index-meta.plist") == 0) { + repo->idxmeta = dict; + } else if (strcmp(filename, "files.plist") == 0) { + repo->files = dict; + } else { + xbps_dbg_printf("[repo] `%s' failed to internalize archive, unrecognized file '%s`.\n", repofile, filename); + /* broken archive, remove it */ + (void)unlink(repofile); + return false; + } + } + + if (repo->idx == NULL) { + xbps_dbg_printf("[repo] `%s' failed to internalize index on archive, removing file.\n", repofile); /* broken archive, remove it */ (void)unlink(repofile); return false; } xbps_dictionary_make_immutable(repo->idx); - repo->idxmeta = repo_get_dict(repo); + if (repo->idxmeta != NULL) { repo->is_signed = true; xbps_dictionary_make_immutable(repo->idxmeta); } + + if (repo->files != NULL) { + xbps_dictionary_make_immutable(repo->files); + } /* * We don't need the archive anymore, we are only * interested in the proplib dictionaries. From 4ac98972d8de22881213623424cbb64fa5e93385 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Friedel=20Sch=C3=B6n?= Date: Mon, 22 Jan 2024 22:55:37 +0100 Subject: [PATCH 02/12] lib,bin: re-implementing xlocate into xbps-{rindex,query} --- bin/xbps-locate/Makefile | 7 - bin/xbps-locate/main.c | 256 ------------------------- bin/xbps-query/Makefile | 2 +- bin/xbps-query/defs.h | 1 + bin/xbps-query/main.c | 36 +++- bin/xbps-query/ownedby.c | 317 ++++++++++++++++-------------- bin/xbps-rindex/Makefile | 2 +- bin/xbps-rindex/defs.h | 7 +- bin/xbps-rindex/files-add.c | 352 ++++++++++++++++++++++++++++++++++ bin/xbps-rindex/index-add.c | 62 +----- bin/xbps-rindex/index-clean.c | 2 +- bin/xbps-rindex/main.c | 6 +- bin/xbps-rindex/repoflush.c | 33 ++-- bin/xbps-rindex/sign.c | 2 +- include/xbps.h.in | 13 ++ include/xbps_api_impl.h | 2 +- lib/Makefile | 2 +- lib/repo.c | 74 ++++--- lib/repo_sync.c | 36 ---- lib/repo_sync_files.c | 118 ++++++++++++ 20 files changed, 756 insertions(+), 574 deletions(-) delete mode 100644 bin/xbps-locate/Makefile delete mode 100644 bin/xbps-locate/main.c create mode 100644 bin/xbps-rindex/files-add.c create mode 100644 lib/repo_sync_files.c diff --git a/bin/xbps-locate/Makefile b/bin/xbps-locate/Makefile deleted file mode 100644 index 067a72539..000000000 --- a/bin/xbps-locate/Makefile +++ /dev/null @@ -1,7 +0,0 @@ -TOPDIR = ../.. --include $(TOPDIR)/config.mk - -BIN = xbps-locate -OBJS = main.o - -include $(TOPDIR)/mk/prog.mk diff --git a/bin/xbps-locate/main.c b/bin/xbps-locate/main.c deleted file mode 100644 index 57f98482d..000000000 --- a/bin/xbps-locate/main.c +++ /dev/null @@ -1,256 +0,0 @@ -/*- - * Copyright (c) 2008-2015 Juan Romero Pardines. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - - -struct locate { - const char* expr; - bool case_ignore; - regex_t regex; -}; - -static void __attribute__((noreturn)) -usage(bool fail) -{ - fprintf(stdout, "Usage: xbps-locate [OPTIONS] file-pattern...\n" - "\n" - "OPTIONS\n" - " -C, --config Path to confdir (xbps.d)\n" - " -c, --cachedir Path to cachedir\n" - " -d, --debug Debug mode shown to stderr\n" - " -h, --help Show usage\n" - " -e, --regex Use extended regular expression pattern\n" - " -i, --ignore-conf-repos Ignore repositories defined in xbps.d\n" - " -I, --ignore-case Match case insensitive\n" - " -M, --memory-sync Remote repository data is fetched and stored\n" - " in memory, ignoring on-disk repodata archives\n" - " -R, --repository Add repository to the top of the list\n" - " This option can be specified multiple times\n" - " -r, --rootdir Full path to rootdir\n" - " -V, --version Show XBPS version\n" - " -v, --verbose Verbose messages\n"); - - exit(fail ? EXIT_FAILURE : EXIT_SUCCESS); -} - -#define LOWERWHEN(ignorecase, chr) (!(ignorecase) ? (chr) : tolower(chr)) - -static bool strcontains(const char* haystack, const char* needle, bool ignorecase) { - const char *a, *b; - - b = needle; - for ( ; *haystack != 0; haystack++) { - if (*haystack != *b) { - continue; - } - a = haystack; - do { - if (*b == 0) - return true; - } while (LOWERWHEN(ignorecase, *a++) == LOWERWHEN(ignorecase, *b++)); - b = needle; - } - return false; -} - -static int repo_search_files(struct xbps_repo* repo, void* locate_ptr, bool* done) { - struct locate *locate = locate_ptr; - const char *pkgname, *file; - xbps_object_iterator_t iter; - xbps_object_t pkgkey; - xbps_array_t pkgfiles; - - (void) done; - - if (repo->files == NULL) { - xbps_warn_printf("repository %s has no files-entry, skipping.\n", repo->uri); - return 1; - } - - iter = xbps_dictionary_iterator(repo->files); - - while ((pkgkey = xbps_object_iterator_next(iter)) != NULL) { - pkgname = xbps_dictionary_keysym_cstring_nocopy(pkgkey); - pkgfiles = xbps_dictionary_get_keysym(repo->files, pkgkey); - - for (unsigned int i = 0; i < xbps_array_count(pkgfiles); i++) { - if (!xbps_array_get_cstring_nocopy(pkgfiles, i, &file)) - continue; - if (locate->expr != NULL) { - if (strcontains(file, locate->expr, locate->case_ignore)) - printf("%s: %s\n", pkgname, file); - } else { - if (!regexec(&locate->regex, file, 0, NULL, 0)) - printf("%s: %s\n", pkgname, file); - } - } - } - - xbps_object_iterator_release(iter); - - return 1; -} - -int -main(int argc, char **argv) -{ - const char *shortopts = "C:c:dehiIMr:R:r:vV"; - const struct option longopts[] = { - { "config", required_argument, NULL, 'C' }, - { "cachedir", required_argument, NULL, 'c' }, - { "debug", no_argument, NULL, 'd' }, - { "regex", no_argument, NULL, 'e' }, - { "help", no_argument, NULL, 'h' }, - { "ignore-conf-repos", no_argument, NULL, 'i' }, - { "ignore-case", no_argument, NULL, 'I' }, - { "memory-sync", no_argument, NULL, 'M' }, - { "regex", no_argument, NULL, 'r' }, - { "repository", required_argument, NULL, 'R' }, - { "rootdir", required_argument, NULL, 'r' }, - { "verbose", no_argument, NULL, 'v' }, - { "version", no_argument, NULL, 'V' }, - { NULL, 0, NULL, 0 }, - }; - - struct xbps_handle xh; - struct locate locate; - const char *rootdir, *cachedir, *confdir; - int c, flags, rv, regflags; - bool regex; - - rootdir = cachedir = confdir = NULL; - flags = rv = c = 0; - regex = false; - regflags = REG_EXTENDED | REG_NOSUB; - - locate.expr = NULL; - locate.case_ignore = false; - - memset(&xh, 0, sizeof(xh)); - - while ((c = getopt_long(argc, argv, shortopts, longopts, NULL)) != -1) { - switch (c) { - case 'C': - confdir = optarg; - break; - case 'c': - cachedir = optarg; - break; - case 'd': - flags |= XBPS_FLAG_DEBUG; - break; - case 'h': - usage(false); - /* NOTREACHED */ - case 'i': - flags |= XBPS_FLAG_IGNORE_CONF_REPOS; - break; - case 'I': - locate.case_ignore = true; - regflags |= REG_ICASE; - break; - case 'M': - flags |= XBPS_FLAG_REPOS_MEMSYNC; - break; - case 'R': - xbps_repo_store(&xh, optarg); - break; - case 'r': - rootdir = optarg; - break; - case 'v': - flags |= XBPS_FLAG_VERBOSE; - break; - case 'V': - printf("%s\n", XBPS_RELVER); - exit(EXIT_SUCCESS); - case 'e': - regex = true; - break; - case '?': - default: - usage(true); - /* NOTREACHED */ - } - } - argc -= optind; - argv += optind; - - if (!argc) { - usage(true); - /* NOTREACHED */ - } - - if (regex) { - if (regcomp(&locate.regex, *(argv++), regflags) != 0) { - xbps_error_printf("invalid regular expression\n"); - exit(1); - } - } else { - locate.expr = *(argv++); - } - argc--; - - if (argc) { - /* trailing parameters */ - usage(true); - /* NOTREACHED */ - } - /* - * Initialize libxbps. - */ - if (rootdir) - xbps_strlcpy(xh.rootdir, rootdir, sizeof(xh.rootdir)); - if (cachedir) - xbps_strlcpy(xh.cachedir, cachedir, sizeof(xh.cachedir)); - if (confdir) - xbps_strlcpy(xh.confdir, confdir, sizeof(xh.confdir)); - - xh.flags = flags; - - if ((rv = xbps_init(&xh)) != 0) { - xbps_error_printf("Failed to initialize libxbps: %s\n", - strerror(rv)); - exit(EXIT_FAILURE); - } - - xbps_rpool_foreach(&xh, repo_search_files, &locate); - - if (regex) - regfree(&locate.regex); - - xbps_end(&xh); - exit(rv); -} diff --git a/bin/xbps-query/Makefile b/bin/xbps-query/Makefile index 23aa3afdc..f2ee56db7 100644 --- a/bin/xbps-query/Makefile +++ b/bin/xbps-query/Makefile @@ -3,6 +3,6 @@ TOPDIR = ../.. BIN = xbps-query OBJS = main.o list.o show-deps.o show-info-files.o -OBJS += ownedby.o search.o ../xbps-install/util.o +OBJS += ownedby.o search.o ../xbps-install/util.o ../xbps-install/fetch_cb.o include $(TOPDIR)/mk/prog.mk diff --git a/bin/xbps-query/defs.h b/bin/xbps-query/defs.h index 2a1747253..5c12564e8 100644 --- a/bin/xbps-query/defs.h +++ b/bin/xbps-query/defs.h @@ -50,6 +50,7 @@ int repo_show_pkg_namedesc(struct xbps_handle *, xbps_object_t, void *, /* from ownedby.c */ int ownedby(struct xbps_handle *, const char *, bool, bool); +int ownedhash(struct xbps_handle *, const char *, bool, bool); /* From list.c */ unsigned int find_longest_pkgver(struct xbps_handle *, xbps_object_t); diff --git a/bin/xbps-query/main.c b/bin/xbps-query/main.c index 44316c1ad..292f81c88 100644 --- a/bin/xbps-query/main.c +++ b/bin/xbps-query/main.c @@ -42,7 +42,8 @@ usage(bool fail) " -c, --cachedir Path to cachedir\n" " -d, --debug Debug mode shown to stderr\n" " -h, --help Show usage\n" - " -i, --ignore-conf-repos Ignore repositories defined in xbps.d\n" + " -F, --update-files Updates the files-database\n" + " -i, --ignore-conf-repos Ignore repositories defined in xbps.d\n" " -M, --memory-sync Remote repository data is fetched and stored\n" " in memory, ignoring on-disk repodata archives\n" " -p, --property PROP[,...] Show properties for PKGNAME\n" @@ -64,7 +65,9 @@ usage(bool fail) " -m, --list-manual-pkgs List packages installed explicitly\n" " -O, --list-orphans List package orphans\n" " -o, --ownedby FILE Search for package files by matching STRING or REGEX\n" - " -S, --show PKG Show information for PKG [default mode]\n" + " --ownedhash FILE Search for package files by matching hash of FILE\n" + " or treating FILE as hash if not present\n" + " -S, --show PKG Show information for PKG [default mode]\n" " -s, --search PKG Search for packages by matching PKG, STRING or REGEX\n" " --cat=FILE PKG Print FILE from PKG binpkg to stdout\n" " -f, --files PKG Show package files for PKG\n" @@ -77,13 +80,14 @@ usage(bool fail) int main(int argc, char **argv) { - const char *shortopts = "C:c:df:hHiLlMmOo:p:Rr:s:S:VvX:x:"; + const char *shortopts = "C:c:dFf:hHiLlMmOo:p:Rr:s:S:VvX:x:"; const struct option longopts[] = { { "config", required_argument, NULL, 'C' }, { "cachedir", required_argument, NULL, 'c' }, { "debug", no_argument, NULL, 'd' }, { "help", no_argument, NULL, 'h' }, { "ignore-conf-repos", no_argument, NULL, 'i' }, + { "update-files", no_argument, NULL, 'F' }, { "list-repos", no_argument, NULL, 'L' }, { "list-pkgs", no_argument, NULL, 'l' }, { "list-hold-pkgs", no_argument, NULL, 'H' }, @@ -92,6 +96,7 @@ main(int argc, char **argv) { "list-manual-pkgs", no_argument, NULL, 'm' }, { "list-orphans", no_argument, NULL, 'O' }, { "ownedby", required_argument, NULL, 'o' }, + { "ownedhash", required_argument, NULL, 4 }, { "property", required_argument, NULL, 'p' }, { "repository", optional_argument, NULL, 'R' }, { "rootdir", required_argument, NULL, 'r' }, @@ -108,17 +113,18 @@ main(int argc, char **argv) { NULL, 0, NULL, 0 }, }; struct xbps_handle xh; + struct xbps_fetch_cb_data xfer; const char *pkg, *rootdir, *cachedir, *confdir, *props, *catfile; int c, flags, rv; - bool list_pkgs, list_repos, orphans, own, list_repolock; + bool list_pkgs, list_repos, orphans, own, ownhash, list_repolock; bool list_manual, list_hold, show_prop, show_files, show_deps, show_rdeps; - bool show, pkg_search, regex, repo_mode, opmode, fulldeptree; + bool show, pkg_search, regex, repo_mode, opmode, fulldeptree, update_files; rootdir = cachedir = confdir = props = pkg = catfile = NULL; flags = rv = c = 0; - list_pkgs = list_repos = list_hold = orphans = pkg_search = own = false; + list_pkgs = list_repos = list_hold = orphans = pkg_search = own = ownhash = false; list_manual = list_repolock = show_prop = show_files = false; - regex = show = show_deps = show_rdeps = fulldeptree = false; + regex = show = show_deps = show_rdeps = fulldeptree = false, update_files = false; repo_mode = opmode = false; memset(&xh, 0, sizeof(xh)); @@ -138,6 +144,9 @@ main(int argc, char **argv) pkg = optarg; show_files = opmode = true; break; + case 'F': + update_files = true; + break; case 'H': list_hold = opmode = true; break; @@ -213,6 +222,10 @@ main(int argc, char **argv) case 3: list_repolock = opmode = true; break; + case 4: + pkg = optarg; + ownhash = opmode = true; + break; case '?': default: usage(true); @@ -247,6 +260,8 @@ main(int argc, char **argv) xbps_strlcpy(xh.confdir, confdir, sizeof(xh.confdir)); xh.flags = flags; + xh.fetch_cb = fetch_file_progress_cb; + xh.fetch_cb_data = &xfer; if ((rv = xbps_init(&xh)) != 0) { xbps_error_printf("Failed to initialize libxbps: %s\n", @@ -254,6 +269,9 @@ main(int argc, char **argv) exit(EXIT_FAILURE); } + if (update_files) + xbps_rpool_sync_files(&xh); + if (list_repos) { /* list repositories */ rv = repo_list(&xh); @@ -282,6 +300,10 @@ main(int argc, char **argv) /* ownedby mode */ rv = ownedby(&xh, pkg, repo_mode, regex); + } else if (ownhash) { + /* ownedby mode */ + rv = ownedhash(&xh, pkg, repo_mode, regex); + } else if (pkg_search) { /* search mode */ rv = search(&xh, repo_mode, pkg, props, regex); diff --git a/bin/xbps-query/ownedby.c b/bin/xbps-query/ownedby.c index baa82c3b4..a155b73c0 100644 --- a/bin/xbps-query/ownedby.c +++ b/bin/xbps-query/ownedby.c @@ -23,185 +23,204 @@ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include -#include +#include "defs.h" +#include "fetch.h" +#include "xbps_api_impl.h" + +#include +#include +#include #include -#include -#include +#include +#include #include -#include #include #include #include - +#include +#include #include -#include "defs.h" -struct ffdata { - bool rematch; - const char *pat, *repouri; - regex_t regex; - xbps_array_t allkeys; - xbps_dictionary_t filesd; + +struct locate { + bool byhash; + const char* expr; + regex_t regex; }; -static void -match_files_by_pattern(xbps_dictionary_t pkg_filesd, - xbps_dictionary_keysym_t key, - struct ffdata *ffd, - const char *pkgver) -{ - xbps_array_t array; - const char *keyname = NULL, *typestr = NULL; - - keyname = xbps_dictionary_keysym_cstring_nocopy(key); - - if (strcmp(keyname, "files") == 0) - typestr = "regular file"; - else if (strcmp(keyname, "links") == 0) - typestr = "link"; - else if (strcmp(keyname, "conf_files") == 0) - typestr = "configuration file"; - else - return; - - array = xbps_dictionary_get_keysym(pkg_filesd, key); - for (unsigned int i = 0; i < xbps_array_count(array); i++) { - xbps_object_t obj; - const char *filestr = NULL, *tgt = NULL; - - obj = xbps_array_get(array, i); - xbps_dictionary_get_cstring_nocopy(obj, "file", &filestr); - if (filestr == NULL) - continue; - xbps_dictionary_get_cstring_nocopy(obj, "target", &tgt); - if (ffd->rematch) { - if (regexec(&ffd->regex, filestr, 0, 0, 0) == 0) { - printf("%s: %s%s%s (%s)\n", - pkgver, filestr, - tgt ? " -> " : "", - tgt ? tgt : "", - typestr); - } - } else { - if ((fnmatch(ffd->pat, filestr, FNM_PERIOD)) == 0) { - printf("%s: %s%s%s (%s)\n", - pkgver, filestr, - tgt ? " -> " : "", - tgt ? tgt : "", - typestr); - } +struct file { + char* hash; + char* path; + char* target; +}; + +static inline void parse_line(struct file* file, char* line) { + char *field, *end_field; + int field_nr; + + field = strtok_r(line, ";", &end_field); + field_nr = 0; + do { + if (field[0] == '%' && field[1] == '\0') + field = NULL; + switch (field_nr++) { + case 0: + file->hash = field; + break; + case 1: + file->path = field; + break; + case 2: + file->target = field; + break; } - } + } while ((field = strtok_r(NULL, ";", &end_field))); } -static int -ownedby_pkgdb_cb(struct xbps_handle *xhp, - xbps_object_t obj, - const char *obj_key UNUSED, - void *arg, - bool *done UNUSED) -{ - xbps_dictionary_t pkgmetad; - xbps_array_t files_keys; - struct ffdata *ffd = arg; - const char *pkgver = NULL; - - (void)obj_key; - (void)done; - - xbps_dictionary_get_cstring_nocopy(obj, "pkgver", &pkgver); - pkgmetad = xbps_pkgdb_get_pkg_files(xhp, pkgver); - if (pkgmetad == NULL) - return 0; - - files_keys = xbps_dictionary_all_keys(pkgmetad); - for (unsigned int i = 0; i < xbps_array_count(files_keys); i++) { - match_files_by_pattern(pkgmetad, - xbps_array_get(files_keys, i), ffd, pkgver); - } - xbps_object_release(pkgmetad); - xbps_object_release(files_keys); +static inline void print_line(const char* pkg, struct file* file) { + printf("%s: %s", pkg, file->path); + if (file->target) + printf(" -> %s", file->target); +} - return 0; +static char* archive_get_file(struct archive* ar, struct archive_entry* entry) { + ssize_t buflen; + char* buf; + + assert(ar != NULL); + assert(entry != NULL); + + buflen = archive_entry_size(entry); + buf = malloc(buflen + 1); + if (buf == NULL) + return NULL; + + if (archive_read_data(ar, buf, buflen) != buflen) { + free(buf); + return NULL; + } + buf[buflen] = '\0'; + return buf; } +static int repo_search_files(struct xbps_repo* repo, void* locate_ptr, bool* done) { + struct locate* locate = locate_ptr; + struct archive* ar; + struct archive_entry* entry; + FILE* ar_file; + char* files_uri; + char* reponame_escaped; + + (void) done; + + if (!xbps_repository_is_remote(repo->uri)) { + files_uri = xbps_xasprintf("%s/%s-files", repo->uri, repo->xhp->target_arch ? repo->xhp->target_arch : repo->xhp->native_arch); + } else { + if (!(reponame_escaped = xbps_get_remote_repo_string(repo->uri))) + return false; + files_uri = xbps_xasprintf("%s/%s/%s-files", repo->xhp->metadir, reponame_escaped, repo->xhp->target_arch ? repo->xhp->target_arch : repo->xhp->native_arch); + free(reponame_escaped); + } -static int -repo_match_cb(struct xbps_handle *xhp, - xbps_object_t obj, - const char *key UNUSED, - void *arg, - bool *done UNUSED) -{ - char bfile[PATH_MAX]; - xbps_dictionary_t filesd; - xbps_array_t files_keys; - struct ffdata *ffd = arg; - const char *pkgver = NULL; - int r; - - xbps_dictionary_set_cstring_nocopy(obj, "repository", ffd->repouri); - xbps_dictionary_get_cstring_nocopy(obj, "pkgver", &pkgver); - - r = xbps_pkg_path_or_url(xhp, bfile, sizeof(bfile), obj); - if (r < 0) { - xbps_error_printf("could not get package path: %s\n", strerror(-r)); - return -r; + if ((ar_file = fopen(files_uri, "r")) == NULL) { + xbps_dbg_printf("[repo] `%s' failed to open file-data archive %s\n", files_uri, strerror(errno)); + return false; } - filesd = xbps_archive_fetch_plist(bfile, "/files.plist"); - if (!filesd) { - xbps_error_printf("%s: couldn't fetch files.plist from %s: %s\n", - pkgver, bfile, strerror(errno)); - return EINVAL; + + ar = archive_read_new(); + assert(ar); + + archive_read_support_filter_gzip(ar); + archive_read_support_filter_bzip2(ar); + archive_read_support_filter_xz(ar); + archive_read_support_filter_lz4(ar); + archive_read_support_filter_zstd(ar); + archive_read_support_format_tar(ar); + + if (archive_read_open_FILE(ar, ar_file) == ARCHIVE_FATAL) { + xbps_dbg_printf("[repo] `%s' failed to open repodata archive %s\n", files_uri, archive_error_string(ar)); + archive_read_close(ar); + fclose(ar_file); + return false; } - files_keys = xbps_dictionary_all_keys(filesd); - for (unsigned int i = 0; i < xbps_array_count(files_keys); i++) { - match_files_by_pattern(filesd, - xbps_array_get(files_keys, i), ffd, pkgver); + + while (archive_read_next_header(ar, &entry) == ARCHIVE_OK) { + const char* pkg = archive_entry_pathname(entry); + char* content = archive_get_file(ar, entry); + char * line, *end_line; + struct file file; + + line = strtok_r(content, "\n", &end_line); + do { + parse_line(&file, line); + if (locate->byhash) { + if (file.hash != NULL && strcasecmp(file.hash, locate->expr) == 0) + print_line(pkg, &file); + } else if (locate->expr != NULL) { + if (strcasestr(file.path, locate->expr) != NULL || (file.target && strcasestr(file.target, locate->expr))) { + print_line(pkg, &file); + } + } else { // regex + if (!regexec(&locate->regex, file.path, 0, NULL, 0) || + (file.target && !regexec(&locate->regex, file.target, 0, NULL, 0))) { + print_line(pkg, &file); + } + } + } while ((line = strtok_r(NULL, "\n", &end_line))); + + free(content); } - xbps_object_release(files_keys); - xbps_object_release(filesd); + fclose(ar_file); + free(files_uri); return 0; } -static int -repo_ownedby_cb(struct xbps_repo *repo, void *arg, bool *done UNUSED) -{ - xbps_array_t allkeys; - struct ffdata *ffd = arg; - int rv; +int ownedby(struct xbps_handle* xh, const char* file, bool repo_mode UNUSED, bool regex) { + struct locate locate; + + memset(&locate, 0, sizeof(locate)); + + if (regex) { + if (regcomp(&locate.regex, file, REG_EXTENDED | REG_NOSUB | REG_ICASE) != 0) { + xbps_error_printf("xbps-locate: invalid regular expression\n"); + exit(1); + } + } else { + locate.expr = file; + } - ffd->repouri = repo->uri; - allkeys = xbps_dictionary_all_keys(repo->idx); - rv = xbps_array_foreach_cb_multi(repo->xhp, allkeys, repo->idx, repo_match_cb, ffd); - xbps_object_release(allkeys); + xbps_rpool_foreach(xh, repo_search_files, &locate); - return rv; + if (regex) + regfree(&locate.regex); + + return 0; } -int -ownedby(struct xbps_handle *xhp, const char *pat, bool repo, bool regex) -{ - struct ffdata ffd; - int rv; +int ownedhash(struct xbps_handle* xh, const char* file, bool repo_mode UNUSED, bool regex UNUSED) { + struct locate locate; + struct stat fstat; + char hash[XBPS_SHA256_SIZE]; - ffd.rematch = false; - ffd.pat = pat; + memset(&locate, 0, sizeof(locate)); + locate.byhash = true; - if (regex) { - ffd.rematch = true; - if (regcomp(&ffd.regex, ffd.pat, REG_EXTENDED|REG_NOSUB|REG_ICASE) != 0) - return EINVAL; + if (stat(file, &fstat) != -1) { + if (!S_ISREG(fstat.st_mode)) { + xbps_error_printf("query: `%s' exist but is not a regular file\n", file); + return 1; + } + if (!xbps_file_sha256(hash, sizeof(hash), file)) { + xbps_error_printf("query: cannot get hash of `%s': %s\n", file, strerror(errno)); + return 1; + } + locate.expr = hash; + } else { + locate.expr = file; } - if (repo) - rv = xbps_rpool_foreach(xhp, repo_ownedby_cb, &ffd); - else - rv = xbps_pkgdb_foreach_cb(xhp, ownedby_pkgdb_cb, &ffd); - if (regex) - regfree(&ffd.regex); + xbps_rpool_foreach(xh, repo_search_files, &locate); - return rv; + return 0; } diff --git a/bin/xbps-rindex/Makefile b/bin/xbps-rindex/Makefile index 6dc200fb4..5e62569ce 100644 --- a/bin/xbps-rindex/Makefile +++ b/bin/xbps-rindex/Makefile @@ -2,7 +2,7 @@ TOPDIR = ../.. -include $(TOPDIR)/config.mk BIN = xbps-rindex -OBJS = main.o index-add.o index-clean.o remove-obsoletes.o repoflush.o sign.o +OBJS = main.o index-add.o files-add.o index-clean.o remove-obsoletes.o repoflush.o sign.o include $(TOPDIR)/mk/prog.mk diff --git a/bin/xbps-rindex/defs.h b/bin/xbps-rindex/defs.h index 2c70fcf5d..974bc2584 100644 --- a/bin/xbps-rindex/defs.h +++ b/bin/xbps-rindex/defs.h @@ -33,6 +33,9 @@ /* From index-add.c */ int index_add(struct xbps_handle *, int, int, char **, bool, const char *); +/* From files-add.c */ +int files_add(struct xbps_handle *, int, int, char **, bool, const char *); + /* From index-clean.c */ int index_clean(struct xbps_handle *, const char *, bool, const char *); @@ -45,7 +48,7 @@ int sign_repo(struct xbps_handle *, const char *, const char *, int sign_pkgs(struct xbps_handle *, int, int, char **, const char *, bool); /* From repoflush.c */ -bool repodata_flush(struct xbps_handle *, const char *, const char *, - xbps_dictionary_t, xbps_dictionary_t, xbps_dictionary_t, const char *); +bool repodata_flush(struct xbps_handle *, const char *, const char *, xbps_dictionary_t, xbps_dictionary_t, const char *); +bool repodata_flush_files(struct xbps_handle *, const char *, const char *, const char *); #endif /* !_XBPS_RINDEX_DEFS_H_ */ diff --git a/bin/xbps-rindex/files-add.c b/bin/xbps-rindex/files-add.c new file mode 100644 index 000000000..27f3b5f18 --- /dev/null +++ b/bin/xbps-rindex/files-add.c @@ -0,0 +1,352 @@ +/*- + * Copyright (c) 2012-2015 Juan Romero Pardines. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "defs.h" +#include "xbps/xbps_array.h" +#include "xbps/xbps_dictionary.h" +#include "xbps/xbps_object.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static inline void +add_files_to_file(xbps_string_t dest, xbps_array_t files) { + xbps_dictionary_t pkg; + xbps_string_t pkg_file, pkg_target, pkg_sha256; + + // 'links' is not always present, so `files` could be NULL + if (files == NULL) + return; + + for (unsigned int i = 0; i < xbps_array_count(files); i++) { + pkg = xbps_array_get(files, i); + pkg_file = xbps_dictionary_get(pkg, "file"); + pkg_target = xbps_dictionary_get(pkg, "target"); + pkg_sha256 = xbps_dictionary_get(pkg, "sha256"); + + if (pkg_sha256) + xbps_string_append(dest, pkg_sha256); + else + xbps_string_append_cstring(dest, "%"); + + xbps_string_append_cstring(dest, ";"); + xbps_string_append(dest, pkg_file); + xbps_string_append_cstring(dest, ";"); + + if (pkg_target) + xbps_string_append(dest, pkg_target); + else + xbps_string_append_cstring(dest, "%"); + + xbps_string_append_cstring(dest, "\n"); + } +} + +static void +list_packages(xbps_array_t dest, struct archive* ar) { + struct archive_entry* entry; + const char* path; + + while (archive_read_next_header(ar, &entry) == ARCHIVE_OK) { + path = archive_entry_pathname(entry); + if (strcmp(path, "HASHES") == 0) + continue; + xbps_array_add_cstring(dest, path); + archive_read_data_skip(ar); + } +} + +static const char* match_pkgname_in_array(xbps_array_t array, const char* str) { + xbps_object_iterator_t iter; + xbps_object_t obj; + const char* pkgdep = NULL; + char pkgname[XBPS_NAME_SIZE]; + + assert(xbps_object_type(array) == XBPS_TYPE_ARRAY); + assert(str != NULL); + + iter = xbps_array_iterator(array); + assert(iter); + + while ((obj = xbps_object_iterator_next(iter))) { + /* match by pkgname against pkgver */ + pkgdep = xbps_string_cstring_nocopy(obj); + if (!xbps_pkg_name(pkgname, XBPS_NAME_SIZE, pkgdep)) + break; + if (strcmp(pkgname, str) == 0) { + xbps_object_iterator_release(iter); + return pkgdep; + } + } + + xbps_object_iterator_release(iter); + return NULL; +} + +int files_add(struct xbps_handle* xhp, int args, int argmax, char** argv, bool force, const char* compression) { + xbps_dictionary_t props_plist, files_plist; + xbps_array_t existing_files, ignore_packages; + char * tmprepodir = NULL, *repodir = NULL, *new_ar_path, *rlockfname = NULL, *files_uri = NULL; + int rv = 0, ret = 0, rlockfd = -1; + FILE* old_ar_file; + int new_ar_file; + struct archive * old_ar = NULL, *new_ar; + struct archive_entry* entry; + mode_t mask; + + existing_files = xbps_array_create(); + ignore_packages = xbps_array_create(); + + assert(argv); + /* + * Read the repository data or create index dictionaries otherwise. + */ + if ((tmprepodir = strdup(argv[args])) == NULL) + return ENOMEM; + + repodir = dirname(tmprepodir); + if (!xbps_repo_lock(xhp, repodir, &rlockfd, &rlockfname)) { + xbps_error_printf("xbps-rindex: cannot lock repository " + "%s: %s\n", + repodir, strerror(errno)); + rv = -1; + goto out; + } + + files_uri = xbps_repo_path_with_name(xhp, repodir, "files"); + if ((old_ar_file = fopen(files_uri, "r")) == NULL) { + if (errno != ENOENT) { + xbps_error_printf("[repo] `%s' failed to open archive %s\n", files_uri, strerror(errno)); + goto out; + } + } else { + old_ar = archive_read_new(); + assert(old_ar); + + archive_read_support_filter_gzip(old_ar); + archive_read_support_filter_bzip2(old_ar); + archive_read_support_filter_xz(old_ar); + archive_read_support_filter_lz4(old_ar); + archive_read_support_filter_zstd(old_ar); + archive_read_support_format_tar(old_ar); + + if (archive_read_open_FILE(old_ar, old_ar_file) == ARCHIVE_FATAL) { + xbps_dbg_printf("[repo] `%s' failed to open repodata archive %s\n", files_uri, archive_error_string(old_ar)); + return false; + } + + list_packages(existing_files, old_ar); + // maybe reopen? + fseek(old_ar_file, 0, SEEK_SET); + } + + new_ar_path = xbps_xasprintf("%s.XXXXXXXXXX", files_uri); + assert(new_ar_path); + mask = umask(S_IXUSR | S_IRWXG | S_IRWXO); + if ((new_ar_file = mkstemp(new_ar_path)) == -1) + return false; + + umask(mask); + + new_ar = archive_write_new(); + assert(new_ar); + + if (compression == NULL || strcmp(compression, "zstd") == 0) { + archive_write_add_filter_zstd(new_ar); + archive_write_set_options(new_ar, "compression-level=9"); + } else if (strcmp(compression, "gzip") == 0) { + archive_write_add_filter_gzip(new_ar); + archive_write_set_options(new_ar, "compression-level=9"); + } else if (strcmp(compression, "bzip2") == 0) { + archive_write_add_filter_bzip2(new_ar); + archive_write_set_options(new_ar, "compression-level=9"); + } else if (strcmp(compression, "lz4") == 0) { + archive_write_add_filter_lz4(new_ar); + archive_write_set_options(new_ar, "compression-level=9"); + } else if (strcmp(compression, "xz") == 0) { + archive_write_add_filter_xz(new_ar); + archive_write_set_options(new_ar, "compression-level=9"); + } else if (strcmp(compression, "none") == 0) { + /* empty */ + } else { + return false; + } + + archive_write_set_format_pax_restricted(new_ar); + if (archive_write_open_fd(new_ar, new_ar_file) != ARCHIVE_OK) + return false; + + /* + * Process all packages specified in argv. + */ + for (int i = args; i < argmax; i++) { + const char * arch = NULL, *pkg = argv[i], *dbpkgver = NULL, *pkgname = NULL, *pkgver = NULL; + xbps_string_t file_content; + xbps_array_t keys; + + assert(pkg); + /* + * Read metadata props plist dictionary from binary package. + */ + if ((props_plist = xbps_archive_fetch_plist(pkg, "/props.plist")) == NULL) { + xbps_error_printf("index: failed to read %s metadata for `%s', skipping!\n", XBPS_PKGPROPS, pkg); + continue; + } + + xbps_dictionary_get_cstring_nocopy(props_plist, "architecture", &arch); + xbps_dictionary_get_cstring_nocopy(props_plist, "pkgver", &pkgver); + if (!xbps_pkg_arch_match(xhp, arch, NULL)) { + fprintf(stderr, "index: skipping %s, unmatched arch (%s)\n", pkgver, arch); + xbps_object_release(props_plist); + } + + xbps_dictionary_get_cstring_nocopy(props_plist, "pkgname", &pkgname); + if (!force && (dbpkgver = match_pkgname_in_array(existing_files, pkgname))) { + /* Only check version if !force */ + ret = xbps_cmpver(pkgver, dbpkgver); + + /* + * If the considered package reverts the package in the index, + * consider the current package as the newer one. + */ + if (ret < 0 && xbps_pkg_reverts(props_plist, dbpkgver)) { + ret = 1; + /* + * If package in the index reverts considered package, consider the + * package in the index as the newer one. + */ + } else if (ret > 0 && xbps_pkg_reverts(props_plist, pkgver)) { + ret = -1; + } + + /* Same version or index version greater */ + if (ret <= 0) { + fprintf(stderr, "files: skipping `%s' (%s), already registered.\n", pkgver, arch); + xbps_object_release(props_plist); + continue; + } + + xbps_array_add_cstring(ignore_packages, dbpkgver); + + printf("files: updating `%s' -> `%s'\n", dbpkgver, pkgver); + } + + if ((files_plist = xbps_archive_fetch_plist(pkg, "/files.plist")) == NULL) { + xbps_error_printf("files: failed to read files.plist metadata for `%s', skipping!\n", pkg); + xbps_object_release(props_plist); + } + + file_content = xbps_string_create(); + keys = xbps_dictionary_all_keys(files_plist); + for (unsigned int j = 0; j < xbps_array_count(keys); j++) { +// xbps_array_t key = xbps_dictionary_get_keysym(files_plist, xbps_array_get(keys, j)); +// printf("key: %d\n", xbps_object_type(key)); + add_files_to_file(file_content, xbps_dictionary_get_keysym(files_plist, xbps_array_get(keys, j))); + } + xbps_object_release(keys); + + xbps_archive_append_buf(new_ar, + xbps_string_cstring_nocopy(file_content), xbps_string_size(file_content), + pkgver, 0644, "root", "root"); + + printf("files: registered `%s'\n", pkgver); + + xbps_object_release(file_content); + xbps_object_release(props_plist); + xbps_object_release(files_plist); + } + + if (old_ar != NULL) { + while (archive_read_next_header(old_ar, &entry) == ARCHIVE_OK) { + char* buffer; + size_t buffer_size; + + if (xbps_match_string_in_array(ignore_packages, archive_entry_pathname(entry))) + continue; + + if (archive_write_header(new_ar, entry) != ARCHIVE_OK) { + archive_entry_free(entry); + continue; + } + + buffer_size = archive_entry_size(entry); + buffer = malloc(buffer_size); + archive_read_data(old_ar, buffer, buffer_size); + + if (archive_write_data(new_ar, buffer, buffer_size) != ARCHIVE_OK) { + archive_entry_free(entry); + free(buffer); + continue; + } + + if (archive_write_finish_entry(new_ar) != ARCHIVE_OK) { + archive_entry_free(entry); + free(buffer); + continue; + } + archive_entry_free(entry); + } + + archive_read_free(old_ar); + } + + /* Write data to tempfile and rename */ + if (archive_write_close(new_ar) != ARCHIVE_OK) + return false; + if (archive_write_free(new_ar) != ARCHIVE_OK) + return false; + + if (fchmod(new_ar_file, 0664) == -1) { + close(new_ar_file); + unlink(new_ar_path); + goto out; + } + close(new_ar_file); + if (rename(new_ar_path, files_uri) == -1) { + unlink(new_ar_path); + goto out; + } + +out: + xbps_repo_unlock(rlockfd, rlockfname); + + if (tmprepodir) + free(tmprepodir); + + return rv; +} diff --git a/bin/xbps-rindex/index-add.c b/bin/xbps-rindex/index-add.c index ecb306975..00b3132c9 100644 --- a/bin/xbps-rindex/index-add.c +++ b/bin/xbps-rindex/index-add.c @@ -41,7 +41,7 @@ static bool repodata_commit(struct xbps_handle *xhp, const char *repodir, - xbps_dictionary_t idx, xbps_dictionary_t meta,xbps_dictionary_t files, xbps_dictionary_t stage, + xbps_dictionary_t idx, xbps_dictionary_t meta, xbps_dictionary_t stage, const char *compression) { xbps_object_iterator_t iter; @@ -174,7 +174,7 @@ repodata_commit(struct xbps_handle *xhp, const char *repodir, printf("stage: added `%s' (%s)\n", pkgver, arch); } xbps_object_iterator_release(iter); - rv = repodata_flush(xhp, repodir, "stagedata", stage, NULL, files, compression); + rv = repodata_flush(xhp, repodir, "stagedata", stage, NULL, compression); } else { char *stagefile; @@ -192,52 +192,17 @@ repodata_commit(struct xbps_handle *xhp, const char *repodir, stagefile = xbps_repo_path_with_name(xhp, repodir, "stagedata"); unlink(stagefile); free(stagefile); - rv = repodata_flush(xhp, repodir, "repodata", idx, meta, files, compression); + rv = repodata_flush(xhp, repodir, "repodata", idx, meta, compression); } xbps_object_release(usedshlibs); xbps_object_release(oldshlibs); return rv; } -static xbps_array_t create_file_list(const char* pkg, xbps_dictionary_t filepkg) { - xbps_array_t file_list = xbps_array_create(); - xbps_array_t files; - const char* path; - - if ((files = xbps_dictionary_get(filepkg, "dirs")) == NULL) { - xbps_error_printf("index: failed to get `dirs' from files.plist in `%s'\n", pkg); - xbps_object_release(file_list); - return NULL; - } - - for (unsigned int i = 0; i < xbps_array_count(files); i++) { - if (!xbps_dictionary_get_cstring_nocopy(xbps_array_get(files, i), "file", &path)) - continue; - - xbps_array_add_cstring(file_list, path); - } - - if ((files = xbps_dictionary_get(filepkg, "files")) == NULL) { - xbps_error_printf("index: failed to get `files' from files.plist in `%s'\n", pkg); - xbps_object_release(file_list); - return NULL; - } - - for (unsigned int i = 0; i < xbps_array_count(files); i++) { - if (!xbps_dictionary_get_cstring_nocopy(xbps_array_get(files, i), "file", &path)) - continue; - - xbps_array_add_cstring(file_list, path); - } - - return file_list; -} - int index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force, const char *compression) { - xbps_dictionary_t idx, idxmeta, idxstage, files, binpkgd, curpkgd, filespkgd; - xbps_array_t file_list; + xbps_dictionary_t idx, idxmeta, idxstage, binpkgd, curpkgd; struct xbps_repo *repo = NULL, *stage = NULL; struct stat st; char *tmprepodir = NULL, *repodir = NULL, *rlockfname = NULL; @@ -267,11 +232,9 @@ index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force if (repo) { idx = xbps_dictionary_copy_mutable(repo->idx); idxmeta = xbps_dictionary_copy_mutable(repo->idxmeta); - files = xbps_dictionary_copy_mutable(repo->files); } else { idx = xbps_dictionary_create(); idxmeta = NULL; - files = xbps_dictionary_create(); } stage = xbps_repo_stage_open(xhp, repodir); if (stage == NULL && errno != ENOENT) { @@ -305,18 +268,11 @@ index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force "`%s', skipping!\n", XBPS_PKGPROPS, pkg); continue; } - filespkgd = xbps_archive_fetch_plist(pkg, "/files.plist"); - if (filespkgd == NULL) { - xbps_warn_printf("index: failed to read files.plist metadata for " - "`%s'!\n", pkg); - } xbps_dictionary_get_cstring_nocopy(binpkgd, "architecture", &arch); xbps_dictionary_get_cstring(binpkgd, "pkgver", &pkgver); if (!xbps_pkg_arch_match(xhp, arch, NULL)) { fprintf(stderr, "index: ignoring %s, unmatched arch (%s)\n", pkgver, arch); xbps_object_release(binpkgd); - if (filespkgd) - xbps_object_release(filespkgd); free(pkgver); continue; } @@ -407,14 +363,6 @@ index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force xbps_dictionary_remove(binpkgd, "version"); xbps_dictionary_remove(binpkgd, "packaged-with"); - if ((file_list = create_file_list(pkg, filespkgd)) == NULL) { - xbps_warn_printf("index: failed to create filelist for `%s', skipping.\n", pkg); - } else { - xbps_dictionary_set(files, pkgname, file_list); - } - xbps_object_release(filespkgd); - xbps_object_release(file_list); - /* * Add new pkg dictionary into the stage index */ @@ -430,7 +378,7 @@ index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force /* * Generate repository data files. */ - if (!repodata_commit(xhp, repodir, idx, idxmeta, files, idxstage, compression)) { + if (!repodata_commit(xhp, repodir, idx, idxmeta, idxstage, compression)) { xbps_error_printf("%s: failed to write repodata: %s\n", _XBPS_RINDEX, strerror(errno)); goto out; diff --git a/bin/xbps-rindex/index-clean.c b/bin/xbps-rindex/index-clean.c index d415cd887..2226e19f7 100644 --- a/bin/xbps-rindex/index-clean.c +++ b/bin/xbps-rindex/index-clean.c @@ -116,7 +116,7 @@ cleanup_repo(struct xbps_handle *xhp, const char *repodir, struct xbps_repo *rep free(stagefile); } if (!xbps_dictionary_equals(dest, repo->idx)) { - if (!repodata_flush(xhp, repodir, reponame, dest, repo->idxmeta, repo->files, compression)) { + if (!repodata_flush(xhp, repodir, reponame, dest, repo->idxmeta, compression)) { rv = errno; xbps_error_printf("failed to write repodata: %s\n", strerror(errno)); diff --git a/bin/xbps-rindex/main.c b/bin/xbps-rindex/main.c index da80d3d00..d1732685a 100644 --- a/bin/xbps-rindex/main.c +++ b/bin/xbps-rindex/main.c @@ -159,9 +159,11 @@ main(int argc, char **argv) exit(EXIT_FAILURE); } - if (add_mode) + if (add_mode) { rv = index_add(&xh, optind, argc, argv, force, compression); - else if (clean_mode) + if (rv == 0) + rv = files_add(&xh, optind, argc, argv, force, compression); + } else if (clean_mode) rv = index_clean(&xh, argv[optind], hashcheck, compression); else if (rm_mode) rv = remove_obsoletes(&xh, argv[optind]); diff --git a/bin/xbps-rindex/repoflush.c b/bin/xbps-rindex/repoflush.c index e43f5739d..010c25919 100644 --- a/bin/xbps-rindex/repoflush.c +++ b/bin/xbps-rindex/repoflush.c @@ -41,11 +41,10 @@ #include #include "defs.h" -#include "xbps/xbps_dictionary.h" bool repodata_flush(struct xbps_handle *xhp, const char *repodir, - const char *reponame, xbps_dictionary_t idx, xbps_dictionary_t meta, xbps_dictionary_t files, + const char *reponame, xbps_dictionary_t idx, xbps_dictionary_t meta, const char *compression) { struct archive *ar; @@ -107,28 +106,20 @@ repodata_flush(struct xbps_handle *xhp, const char *repodir, if (rv != 0) return false; - if (meta != NULL) { /* XBPS_REPOIDX_META */ - buf = xbps_dictionary_externalize(meta); + /* XBPS_REPOIDX_META */ + if (meta == NULL) { + /* fake entry */ + buf = strdup("DEADBEEF"); if (buf == NULL) return false; - rv = xbps_archive_append_buf(ar, buf, strlen(buf), - XBPS_REPOIDX_META, 0644, "root", "root"); - free(buf); - if (rv != 0) - return false; - } - - if (files != NULL) { - /* XBPS_REPOIDX_META */ - buf = xbps_dictionary_externalize(files); - if (buf == NULL) - return false; - rv = xbps_archive_append_buf(ar, buf, strlen(buf), - XBPS_REPO_FILES, 0644, "root", "root"); - free(buf); - if (rv != 0) - return false; + } else { + buf = xbps_dictionary_externalize(meta); } + rv = xbps_archive_append_buf(ar, buf, strlen(buf), + XBPS_REPOIDX_META, 0644, "root", "root"); + free(buf); + if (rv != 0) + return false; /* Write data to tempfile and rename */ if (archive_write_close(ar) != ARCHIVE_OK) diff --git a/bin/xbps-rindex/sign.c b/bin/xbps-rindex/sign.c index 7179ba1bd..7e6cd2ce2 100644 --- a/bin/xbps-rindex/sign.c +++ b/bin/xbps-rindex/sign.c @@ -222,7 +222,7 @@ sign_repo(struct xbps_handle *xhp, const char *repodir, _XBPS_RINDEX, strerror(errno)); goto out; } - flush_failed = repodata_flush(xhp, repodir, "repodata", repo->idx, meta, repo->files, compression); + flush_failed = repodata_flush(xhp, repodir, "repodata", repo->idx, meta, compression); xbps_repo_unlock(rlockfd, rlockfname); if (!flush_failed) { xbps_error_printf("failed to write repodata: %s\n", strerror(errno)); diff --git a/include/xbps.h.in b/include/xbps.h.in index cb992ab7f..70b6beafb 100644 --- a/include/xbps.h.in +++ b/include/xbps.h.in @@ -1454,6 +1454,12 @@ struct xbps_repo { * @private */ int fd; + /** + * @var filear + * + * archive_read instance for *-files + */ + struct archive *filear; /** * var is_remote * @@ -1483,6 +1489,8 @@ void xbps_rpool_release(struct xbps_handle *xhp); */ int xbps_rpool_sync(struct xbps_handle *xhp, const char *uri); +int xbps_rpool_sync_files(struct xbps_handle* xhp); + /** * Iterates over the repository pool and executes the \a fn function * callback passing in the void * \a arg argument to it. The bool pointer @@ -2380,6 +2388,11 @@ xbps_plist_array_from_file(const char *path); xbps_dictionary_t xbps_plist_dictionary_from_file(const char *path); +/** + +*/ +char* xbps_get_remote_repo_string(const char *path); + /**@}*/ #ifdef __cplusplus diff --git a/include/xbps_api_impl.h b/include/xbps_api_impl.h index aac5b11b5..f4a7f8d7f 100644 --- a/include/xbps_api_impl.h +++ b/include/xbps_api_impl.h @@ -99,7 +99,6 @@ int HIDDEN xbps_transaction_fetch(struct xbps_handle *, int HIDDEN xbps_transaction_pkg_deps(struct xbps_handle *, xbps_array_t, xbps_dictionary_t); int HIDDEN xbps_transaction_internalize(struct xbps_handle *, xbps_object_iterator_t); -char HIDDEN *xbps_get_remote_repo_string(const char *); int HIDDEN xbps_repo_sync(struct xbps_handle *, const char *); int HIDDEN xbps_file_hash_check_dictionary(struct xbps_handle *, xbps_dictionary_t, const char *, const char *); @@ -120,5 +119,6 @@ xbps_array_t HIDDEN xbps_get_pkg_fulldeptree(struct xbps_handle *, struct xbps_repo HIDDEN *xbps_regget_repo(struct xbps_handle *, const char *); int HIDDEN xbps_conf_init(struct xbps_handle *); +int HIDDEN xbps_repo_sync_files(struct xbps_handle* xh, const char* uri); #endif /* !_XBPS_API_IMPL_H_ */ diff --git a/lib/Makefile b/lib/Makefile index 0cf6ac84f..481f82cc7 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -48,7 +48,7 @@ OBJS += pubkey2fp.o package_fulldeptree.o OBJS += download.o initend.o pkgdb.o OBJS += plist.o plist_find.o plist_match.o archive.o OBJS += plist_remove.o plist_fetch.o util.o util_path.o util_hash.o -OBJS += repo.o repo_sync.o +OBJS += repo.o repo_sync.o repo_sync_files.o OBJS += rpool.o cb_util.o proplib_wrapper.o OBJS += package_alternatives.o OBJS += conf.o log.o diff --git a/lib/repo.c b/lib/repo.c index 182075692..5452da9aa 100644 --- a/lib/repo.c +++ b/lib/repo.c @@ -40,6 +40,7 @@ #include #include +#include "fetch.h" #include "xbps_api_impl.h" /** @@ -58,14 +59,14 @@ xbps_repo_path_with_name(struct xbps_handle *xhp, const char *url, const char *n { assert(xhp); assert(url); - assert(strcmp(name, "repodata") == 0 || strcmp(name, "stagedata") == 0 || strcmp(name, "files") == 0); + assert(strcmp(name, "repodata") == 0 || strcmp(name, "stagedata") == 0); return xbps_xasprintf("%s/%s-%s", url, xhp->target_arch ? xhp->target_arch : xhp->native_arch, name); } static xbps_dictionary_t -repo_get_dict(struct xbps_repo *repo, const char** filename) +repo_get_dict(struct xbps_repo *repo) { struct archive_entry *entry; int rv; @@ -79,7 +80,6 @@ repo_get_dict(struct xbps_repo *repo, const char** filename) archive_error_string(repo->ar)); return NULL; } - *filename = archive_entry_pathname(entry); return xbps_archive_get_dictionary(repo->ar, entry); } @@ -137,8 +137,6 @@ static bool repo_open_local(struct xbps_repo *repo, const char *repofile) { struct stat st; - const char* filename; - xbps_dictionary_t dict; if (fstat(repo->fd, &st) == -1) { xbps_dbg_printf("[repo] `%s' fstat repodata %s\n", @@ -160,41 +158,19 @@ repo_open_local(struct xbps_repo *repo, const char *repofile) repofile, archive_error_string(repo->ar)); return false; } - - repo->idx = NULL; - repo->idxmeta = NULL; - repo->files = NULL; - while ((dict=repo_get_dict(repo, &filename)) != NULL) { - if (strcmp(filename, "index.plist") == 0) { - repo->idx = dict; - } else if (strcmp(filename, "index-meta.plist") == 0) { - repo->idxmeta = dict; - } else if (strcmp(filename, "files.plist") == 0) { - repo->files = dict; - } else { - xbps_dbg_printf("[repo] `%s' failed to internalize archive, unrecognized file '%s`.\n", repofile, filename); - /* broken archive, remove it */ - (void)unlink(repofile); - return false; - } - } - - if (repo->idx == NULL) { - xbps_dbg_printf("[repo] `%s' failed to internalize index on archive, removing file.\n", repofile); + if ((repo->idx = repo_get_dict(repo)) == NULL) { + xbps_dbg_printf("[repo] `%s' failed to internalize " + " index on archive, removing file.\n", repofile); /* broken archive, remove it */ (void)unlink(repofile); return false; } xbps_dictionary_make_immutable(repo->idx); - + repo->idxmeta = repo_get_dict(repo); if (repo->idxmeta != NULL) { repo->is_signed = true; xbps_dictionary_make_immutable(repo->idxmeta); } - - if (repo->files != NULL) { - xbps_dictionary_make_immutable(repo->files); - } /* * We don't need the archive anymore, we are only * interested in the proplib dictionaries. @@ -732,3 +708,39 @@ xbps_repo_key_import(struct xbps_repo *repo) free(rkeyfile); return rv; } + +char* +xbps_get_remote_repo_string(const char *uri) +{ + struct url *url; + size_t i; + char *p; + + if ((url = fetchParseURL(uri)) == NULL) + return NULL; + + /* + * Replace '.' ':' and '/' characters with underscores, so that + * provided URL: + * + * http://nocturno.local:8080/repo/x86_64 + * + * becomes: + * + * http___nocturno_local_8080_repo_x86_64 + */ + if (url->port != 0) + p = xbps_xasprintf("%s://%s:%u%s", url->scheme, + url->host, url->port, url->doc); + else + p = xbps_xasprintf("%s://%s%s", url->scheme, + url->host, url->doc); + + fetchFreeURL(url); + for (i = 0; i < strlen(p); i++) { + if (p[i] == '.' || p[i] == '/' || p[i] == ':') + p[i] = '_'; + } + + return p; +} \ No newline at end of file diff --git a/lib/repo_sync.c b/lib/repo_sync.c index 2f803a0c4..8eedc2c06 100644 --- a/lib/repo_sync.c +++ b/lib/repo_sync.c @@ -34,42 +34,6 @@ #include "xbps_api_impl.h" #include "fetch.h" -char HIDDEN * -xbps_get_remote_repo_string(const char *uri) -{ - struct url *url; - size_t i; - char *p; - - if ((url = fetchParseURL(uri)) == NULL) - return NULL; - - /* - * Replace '.' ':' and '/' characters with underscores, so that - * provided URL: - * - * http://nocturno.local:8080/repo/x86_64 - * - * becomes: - * - * http___nocturno_local_8080_repo_x86_64 - */ - if (url->port != 0) - p = xbps_xasprintf("%s://%s:%u%s", url->scheme, - url->host, url->port, url->doc); - else - p = xbps_xasprintf("%s://%s%s", url->scheme, - url->host, url->doc); - - fetchFreeURL(url); - for (i = 0; i < strlen(p); i++) { - if (p[i] == '.' || p[i] == '/' || p[i] == ':') - p[i] = '_'; - } - - return p; -} - /* * Returns -1 on error, 0 if transfer was not necessary (local/remote * size and/or mtime match) and 1 if downloaded successfully. diff --git a/lib/repo_sync_files.c b/lib/repo_sync_files.c new file mode 100644 index 000000000..005826bc4 --- /dev/null +++ b/lib/repo_sync_files.c @@ -0,0 +1,118 @@ +/*- + * Copyright (c) 2009-2014 Juan Romero Pardines. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include +#include +#include +#include +#include + +#include "xbps.h" +#include "xbps_api_impl.h" +#include "fetch.h" + + +int HIDDEN xbps_repo_sync_files(struct xbps_handle* xh, const char* uri) { + mode_t prev_umask; + const char *arch, *fetchstr = NULL; + char * repodata, *lrepodir, *uri_fixedp; + int rv = 0; + + /* ignore non remote repositories */ + if (!xbps_repository_is_remote(uri)) + return 0; + + uri_fixedp = xbps_get_remote_repo_string(uri); + if (uri_fixedp == NULL) + return -1; + + if (xh->target_arch) + arch = xh->target_arch; + else + arch = xh->native_arch; + + /* + * Full path to repository directory to store the plist + * index file. + */ + lrepodir = xbps_xasprintf("%s/%s", xh->metadir, uri_fixedp); + free(uri_fixedp); + /* + * Create repodir in metadir. + */ + prev_umask = umask(022); + if ((rv = xbps_mkpath(lrepodir, 0755)) == -1 && errno != EEXIST) { + xbps_error_printf("[reposync] to create repodir `%s': %s\n", lrepodir, strerror(errno)); + umask(prev_umask); + free(lrepodir); + return rv; + } + if (chdir(lrepodir) == -1) { + xbps_error_printf("[reposync] failed to change dir to repodir `%s': %s\n", lrepodir, strerror(errno)); + umask(prev_umask); + free(lrepodir); + return -1; + } + free(lrepodir); + /* + * Remote repository plist index full URL. + */ + repodata = xbps_xasprintf("%s/%s-files", uri, arch); + + /* reposync start cb */ + printf("[*] Updating file-database `%s' ...\n", repodata); + /* + * Download plist index file from repository. + */ + if ((rv = xbps_fetch_file(xh, repodata, NULL)) != 1 && fetchLastErrCode != FETCH_UNCHANGED) { + /* reposync error cb */ + fetchstr = xbps_fetch_error_string(); + + xbps_error_printf("[reposync] failed to fetch file `%s': %s\n", repodata, fetchstr ? fetchstr : strerror(errno)); + } else if (rv == 1) + rv = 0; + umask(prev_umask); + + free(repodata); + + return rv; +} + +int xbps_rpool_sync_files(struct xbps_handle* xhp) { + const char* repouri = NULL; + + for (unsigned int i = 0; i < xbps_array_count(xhp->repositories); i++) { + xbps_array_get_cstring_nocopy(xhp->repositories, i, &repouri); + if (xbps_repo_sync_files(xhp, repouri) == -1) { + xbps_dbg_printf( + "[rpool] `%s' failed to fetch repository data: %s\n", + repouri, fetchLastErrCode == 0 ? strerror(errno) : xbps_fetch_error_string()); + continue; + } + } + return 0; +} \ No newline at end of file From 02bc416572556376bfd3f22bd45366c22be09efa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Friedel=20Sch=C3=B6n?= Date: Tue, 23 Jan 2024 18:15:34 +0100 Subject: [PATCH 03/12] bin: remove xbps-locate from Makefile --- bin/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/Makefile b/bin/Makefile index 417caf345..c0586aabb 100644 --- a/bin/Makefile +++ b/bin/Makefile @@ -14,7 +14,6 @@ SUBDIRS += xbps-checkvers SUBDIRS += xbps-fbulk SUBDIRS += xbps-digest SUBDIRS += xbps-fetch -SUBDIRS += xbps-locate ifeq (${XBPS_OS},linux) SUBDIRS += xbps-uchroot From d755631f665182f3c32d8aa54836a2f08c54b581 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Friedel=20Sch=C3=B6n?= Date: Wed, 24 Jan 2024 17:11:36 +0100 Subject: [PATCH 04/12] bin: optimalizing writing to entry --- bin/xbps-rindex/files-add.c | 152 +++++++++++++++++++++++++----------- 1 file changed, 106 insertions(+), 46 deletions(-) diff --git a/bin/xbps-rindex/files-add.c b/bin/xbps-rindex/files-add.c index 27f3b5f18..d89545776 100644 --- a/bin/xbps-rindex/files-add.c +++ b/bin/xbps-rindex/files-add.c @@ -27,14 +27,18 @@ #include "xbps/xbps_array.h" #include "xbps/xbps_dictionary.h" #include "xbps/xbps_object.h" +#include "xbps/xbps_string.h" #include #include +#include +#include #include #include #include #include #include +#include #include #include #include @@ -44,38 +48,6 @@ #include #include -static inline void -add_files_to_file(xbps_string_t dest, xbps_array_t files) { - xbps_dictionary_t pkg; - xbps_string_t pkg_file, pkg_target, pkg_sha256; - - // 'links' is not always present, so `files` could be NULL - if (files == NULL) - return; - - for (unsigned int i = 0; i < xbps_array_count(files); i++) { - pkg = xbps_array_get(files, i); - pkg_file = xbps_dictionary_get(pkg, "file"); - pkg_target = xbps_dictionary_get(pkg, "target"); - pkg_sha256 = xbps_dictionary_get(pkg, "sha256"); - - if (pkg_sha256) - xbps_string_append(dest, pkg_sha256); - else - xbps_string_append_cstring(dest, "%"); - - xbps_string_append_cstring(dest, ";"); - xbps_string_append(dest, pkg_file); - xbps_string_append_cstring(dest, ";"); - - if (pkg_target) - xbps_string_append(dest, pkg_target); - else - xbps_string_append_cstring(dest, "%"); - - xbps_string_append_cstring(dest, "\n"); - } -} static void list_packages(xbps_array_t dest, struct archive* ar) { @@ -118,18 +90,77 @@ static const char* match_pkgname_in_array(xbps_array_t array, const char* str) { return NULL; } +static inline struct archive_entry* +make_entry(const char* pkgname, size_t size) { + struct archive_entry* entry; + + entry = archive_entry_new(); + if (entry == NULL) + return NULL; + + archive_entry_set_filetype(entry, AE_IFREG); + archive_entry_set_perm(entry, 0644); + archive_entry_set_uname(entry, "root"); + archive_entry_set_gname(entry, "root"); + archive_entry_set_pathname(entry, pkgname); + archive_entry_set_size(entry, size); + + return entry; +} + +static inline int +make_file_string(xbps_dictionary_t pkg, char* buffer, int buffer_size) { + const char *pkg_file, *pkg_target, *pkg_sha256; + int result; + + if (!xbps_dictionary_get_cstring_nocopy(pkg, "file", &pkg_file)) + return errno = EINVAL, 0; + + if (!xbps_dictionary_get_cstring_nocopy(pkg, "target", &pkg_target)) + pkg_target = ""; + + if (!xbps_dictionary_get_cstring_nocopy(pkg, "sha256", &pkg_sha256)) + pkg_sha256 = ""; + + if ((result = snprintf(buffer, buffer_size, "%s;%s;%s\n", pkg_sha256, pkg_file, pkg_target)) >= buffer_size) + return errno = ENOBUFS, 0; + + if (result == -1) + return 0; + + return result; +} + +static inline int +length_file_string(xbps_dictionary_t pkg) { + const char* field; + size_t size = 3; // 2x ';' + '\n' + + if (xbps_dictionary_get_cstring_nocopy(pkg, "file", &field)) + size += strlen(field); + + if (xbps_dictionary_get_cstring_nocopy(pkg, "target", &field)) + size += strlen(field); + + if (xbps_dictionary_get_cstring_nocopy(pkg, "sha256", &field)) + size += strlen(field); + + return size; +} + int files_add(struct xbps_handle* xhp, int args, int argmax, char** argv, bool force, const char* compression) { xbps_dictionary_t props_plist, files_plist; xbps_array_t existing_files, ignore_packages; char * tmprepodir = NULL, *repodir = NULL, *new_ar_path, *rlockfname = NULL, *files_uri = NULL; int rv = 0, ret = 0, rlockfd = -1; + char pkg_line[65536]; FILE* old_ar_file; int new_ar_file; struct archive * old_ar = NULL, *new_ar; struct archive_entry* entry; mode_t mask; - existing_files = xbps_array_create(); + existing_files = xbps_array_create(); ignore_packages = xbps_array_create(); assert(argv); @@ -215,9 +246,10 @@ int files_add(struct xbps_handle* xhp, int args, int argmax, char** argv, bool f * Process all packages specified in argv. */ for (int i = args; i < argmax; i++) { - const char * arch = NULL, *pkg = argv[i], *dbpkgver = NULL, *pkgname = NULL, *pkgver = NULL; - xbps_string_t file_content; - xbps_array_t keys; + const char * arch = NULL, *pkg = argv[i], *dbpkgver = NULL, *pkgname = NULL, *pkgver = NULL; + struct archive_entry* file_entry; + xbps_array_t keys; + size_t total_size = 0; assert(pkg); /* @@ -269,24 +301,52 @@ int files_add(struct xbps_handle* xhp, int args, int argmax, char** argv, bool f if ((files_plist = xbps_archive_fetch_plist(pkg, "/files.plist")) == NULL) { xbps_error_printf("files: failed to read files.plist metadata for `%s', skipping!\n", pkg); xbps_object_release(props_plist); + continue; } - file_content = xbps_string_create(); keys = xbps_dictionary_all_keys(files_plist); for (unsigned int j = 0; j < xbps_array_count(keys); j++) { -// xbps_array_t key = xbps_dictionary_get_keysym(files_plist, xbps_array_get(keys, j)); -// printf("key: %d\n", xbps_object_type(key)); - add_files_to_file(file_content, xbps_dictionary_get_keysym(files_plist, xbps_array_get(keys, j))); + xbps_array_t files = xbps_dictionary_get_keysym(files_plist, xbps_array_get(keys, j)); + + for (unsigned int k = 0; k < xbps_array_count(files); k++) + total_size += length_file_string(xbps_array_get(files, k)); + } + + file_entry = make_entry(pkgver, total_size); + if (archive_write_header(new_ar, file_entry) != ARCHIVE_OK) { + xbps_warn_printf("files: unable to write entry for %s: %s\n", pkgver, archive_error_string(new_ar)); + archive_entry_free(file_entry); + xbps_object_release(props_plist); + xbps_object_release(files_plist); + continue; + } + + for (unsigned int j = 0; j < xbps_array_count(keys); j++) { + int size; + xbps_array_t files = xbps_dictionary_get_keysym(files_plist, xbps_array_get(keys, j)); + + for (unsigned int k = 0; k < xbps_array_count(files); k++) { + if (!(size = make_file_string(xbps_array_get(files, k), pkg_line, sizeof(pkg_line)))) { + xbps_warn_printf("files: unable to create file-entry for %s: %s\n", pkgver, strerror(errno)); + continue; + } + + if (archive_write_data(new_ar, pkg_line, size) == -1) { + xbps_warn_printf("files: unable to file-write entry for %s: %s\n", pkgver, archive_error_string(new_ar)); + continue; + } + } } - xbps_object_release(keys); - xbps_archive_append_buf(new_ar, - xbps_string_cstring_nocopy(file_content), xbps_string_size(file_content), - pkgver, 0644, "root", "root"); + + if (archive_write_finish_entry(new_ar) != ARCHIVE_OK) { + archive_entry_free(file_entry); + exit(archive_errno(new_ar)); + } + archive_entry_free(file_entry); printf("files: registered `%s'\n", pkgver); - xbps_object_release(file_content); xbps_object_release(props_plist); xbps_object_release(files_plist); } @@ -305,7 +365,7 @@ int files_add(struct xbps_handle* xhp, int args, int argmax, char** argv, bool f } buffer_size = archive_entry_size(entry); - buffer = malloc(buffer_size); + buffer = malloc(buffer_size); archive_read_data(old_ar, buffer, buffer_size); if (archive_write_data(new_ar, buffer, buffer_size) != ARCHIVE_OK) { From 48f2c9c449e98736b08087dc4eb32058888f22bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Friedel=20Sch=C3=B6n?= Date: Wed, 24 Jan 2024 17:12:09 +0100 Subject: [PATCH 05/12] lib: add `-files` as valid repo-file --- lib/repo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/repo.c b/lib/repo.c index 5452da9aa..6808f9716 100644 --- a/lib/repo.c +++ b/lib/repo.c @@ -59,7 +59,7 @@ xbps_repo_path_with_name(struct xbps_handle *xhp, const char *url, const char *n { assert(xhp); assert(url); - assert(strcmp(name, "repodata") == 0 || strcmp(name, "stagedata") == 0); + assert(strcmp(name, "repodata") == 0 || strcmp(name, "stagedata") == 0 || strcmp(name, "files") == 0); return xbps_xasprintf("%s/%s-%s", url, xhp->target_arch ? xhp->target_arch : xhp->native_arch, name); From de733e46eeb75a8bc6f2a84af5e34e9db1eef267 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Friedel=20Sch=C3=B6n?= Date: Thu, 25 Jan 2024 13:06:52 +0100 Subject: [PATCH 06/12] bin: remove unused imports --- bin/xbps-rindex/files-add.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/bin/xbps-rindex/files-add.c b/bin/xbps-rindex/files-add.c index d89545776..c7267afa8 100644 --- a/bin/xbps-rindex/files-add.c +++ b/bin/xbps-rindex/files-add.c @@ -24,21 +24,15 @@ */ #include "defs.h" -#include "xbps/xbps_array.h" -#include "xbps/xbps_dictionary.h" -#include "xbps/xbps_object.h" -#include "xbps/xbps_string.h" #include #include -#include -#include #include #include #include #include #include -#include +#include #include #include #include From bae50131c8928a5850dcc81056b9f5d8ef2671dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Friedel=20Sch=C3=B6n?= Date: Thu, 25 Jan 2024 19:11:08 +0100 Subject: [PATCH 07/12] bin: locate: moving from ;-delimited string to unix :-delimited string --- bin/xbps-query/ownedby.c | 75 ++++++++++++++++--------------------- bin/xbps-rindex/files-add.c | 27 +++++++------ 2 files changed, 48 insertions(+), 54 deletions(-) diff --git a/bin/xbps-query/ownedby.c b/bin/xbps-query/ownedby.c index a155b73c0..d6fbb47f3 100644 --- a/bin/xbps-query/ownedby.c +++ b/bin/xbps-query/ownedby.c @@ -48,39 +48,29 @@ struct locate { regex_t regex; }; -struct file { - char* hash; - char* path; - char* target; -}; +static inline int parse_line(char** fields, int fields_size, char* line) { + char* next; + int i = 0; -static inline void parse_line(struct file* file, char* line) { - char *field, *end_field; - int field_nr; - - field = strtok_r(line, ";", &end_field); - field_nr = 0; - do { - if (field[0] == '%' && field[1] == '\0') - field = NULL; - switch (field_nr++) { - case 0: - file->hash = field; - break; - case 1: - file->path = field; - break; - case 2: - file->target = field; - break; - } - } while ((field = strtok_r(NULL, ";", &end_field))); + while (i < fields_size - 1) { + if (!(next = strchr(line, ':'))) + break; + + *next = '\0'; + + fields[i++] = *line ? line : NULL; + line = next + 1; + } + fields[i++] = *line ? line : NULL; + + return i; } -static inline void print_line(const char* pkg, struct file* file) { - printf("%s: %s", pkg, file->path); - if (file->target) - printf(" -> %s", file->target); +static inline void print_line(const char* pkg, char** file) { + printf("%s: %s", pkg, file[1]); + if (file[2]) + printf(" -> %s", file[2]); + printf("\n"); } static char* archive_get_file(struct archive* ar, struct archive_entry* entry) { @@ -147,26 +137,27 @@ static int repo_search_files(struct xbps_repo* repo, void* locate_ptr, bool* don while (archive_read_next_header(ar, &entry) == ARCHIVE_OK) { const char* pkg = archive_entry_pathname(entry); char* content = archive_get_file(ar, entry); - char * line, *end_line; - struct file file; + char* line, *end_line; + char* file[3]; line = strtok_r(content, "\n", &end_line); - do { - parse_line(&file, line); + while (line) { + parse_line(file, 3, line); if (locate->byhash) { - if (file.hash != NULL && strcasecmp(file.hash, locate->expr) == 0) - print_line(pkg, &file); + if (file[0] != NULL && strcasecmp(file[0], locate->expr) == 0) + print_line(pkg, file); } else if (locate->expr != NULL) { - if (strcasestr(file.path, locate->expr) != NULL || (file.target && strcasestr(file.target, locate->expr))) { - print_line(pkg, &file); + if (strcasestr(file[1], locate->expr) != NULL || (file[2] && strcasestr(file[2], locate->expr))) { + print_line(pkg, file); } } else { // regex - if (!regexec(&locate->regex, file.path, 0, NULL, 0) || - (file.target && !regexec(&locate->regex, file.target, 0, NULL, 0))) { - print_line(pkg, &file); + if (!regexec(&locate->regex, file[1], 0, NULL, 0) || + (file[2] && !regexec(&locate->regex, file[2], 0, NULL, 0))) { + print_line(pkg, file); } } - } while ((line = strtok_r(NULL, "\n", &end_line))); + line = strtok_r(NULL, "\n", &end_line); + } free(content); } diff --git a/bin/xbps-rindex/files-add.c b/bin/xbps-rindex/files-add.c index c7267afa8..0c0af2f82 100644 --- a/bin/xbps-rindex/files-add.c +++ b/bin/xbps-rindex/files-add.c @@ -116,9 +116,9 @@ make_file_string(xbps_dictionary_t pkg, char* buffer, int buffer_size) { if (!xbps_dictionary_get_cstring_nocopy(pkg, "sha256", &pkg_sha256)) pkg_sha256 = ""; - if ((result = snprintf(buffer, buffer_size, "%s;%s;%s\n", pkg_sha256, pkg_file, pkg_target)) >= buffer_size) + if ((result = snprintf(buffer, buffer_size, "%s:%s:%s\n", pkg_sha256, pkg_file, pkg_target)) >= buffer_size) return errno = ENOBUFS, 0; - + if (result == -1) return 0; @@ -128,16 +128,19 @@ make_file_string(xbps_dictionary_t pkg, char* buffer, int buffer_size) { static inline int length_file_string(xbps_dictionary_t pkg) { const char* field; - size_t size = 3; // 2x ';' + '\n' + size_t size = 3; // 2x ':' + '\n' - if (xbps_dictionary_get_cstring_nocopy(pkg, "file", &field)) - size += strlen(field); + size += xbps_dictionary_get_cstring_nocopy(pkg, "file", &field) + ? strlen(field) + : 0; - if (xbps_dictionary_get_cstring_nocopy(pkg, "target", &field)) - size += strlen(field); + size += xbps_dictionary_get_cstring_nocopy(pkg, "target", &field) + ? strlen(field) + : 0; - if (xbps_dictionary_get_cstring_nocopy(pkg, "sha256", &field)) - size += strlen(field); + size += xbps_dictionary_get_cstring_nocopy(pkg, "sha256", &field) + ? strlen(field) + : 0; return size; } @@ -243,7 +246,7 @@ int files_add(struct xbps_handle* xhp, int args, int argmax, char** argv, bool f const char * arch = NULL, *pkg = argv[i], *dbpkgver = NULL, *pkgname = NULL, *pkgver = NULL; struct archive_entry* file_entry; xbps_array_t keys; - size_t total_size = 0; + size_t total_size = 0; assert(pkg); /* @@ -303,7 +306,7 @@ int files_add(struct xbps_handle* xhp, int args, int argmax, char** argv, bool f xbps_array_t files = xbps_dictionary_get_keysym(files_plist, xbps_array_get(keys, j)); for (unsigned int k = 0; k < xbps_array_count(files); k++) - total_size += length_file_string(xbps_array_get(files, k)); + total_size += length_file_string(xbps_array_get(files, k)); } file_entry = make_entry(pkgver, total_size); @@ -331,7 +334,7 @@ int files_add(struct xbps_handle* xhp, int args, int argmax, char** argv, bool f } } } - + if (archive_write_finish_entry(new_ar) != ARCHIVE_OK) { archive_entry_free(file_entry); From 401bc01186af130b227b33ec4922bcf5eae53437 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Friedel=20Sch=C3=B6n?= Date: Thu, 25 Jan 2024 19:21:47 +0100 Subject: [PATCH 08/12] bin: replace (void)var to UNUSED --- bin/xbps-query/ownedby.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bin/xbps-query/ownedby.c b/bin/xbps-query/ownedby.c index d6fbb47f3..fe14965ea 100644 --- a/bin/xbps-query/ownedby.c +++ b/bin/xbps-query/ownedby.c @@ -93,7 +93,7 @@ static char* archive_get_file(struct archive* ar, struct archive_entry* entry) { return buf; } -static int repo_search_files(struct xbps_repo* repo, void* locate_ptr, bool* done) { +static int repo_search_files(struct xbps_repo* repo, void* locate_ptr, bool* done UNUSED) { struct locate* locate = locate_ptr; struct archive* ar; struct archive_entry* entry; @@ -101,8 +101,6 @@ static int repo_search_files(struct xbps_repo* repo, void* locate_ptr, bool* don char* files_uri; char* reponame_escaped; - (void) done; - if (!xbps_repository_is_remote(repo->uri)) { files_uri = xbps_xasprintf("%s/%s-files", repo->uri, repo->xhp->target_arch ? repo->xhp->target_arch : repo->xhp->native_arch); } else { From c5b36bde45be6f8b872068920d60202e5327f86e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Friedel=20Sch=C3=B6n?= Date: Mon, 29 Jan 2024 00:13:33 +0100 Subject: [PATCH 09/12] bin: multi-threading locate --- bin/xbps-query/ownedby.c | 208 ++++++++++++++++++++++++------------ bin/xbps-rindex/files-add.c | 2 +- 2 files changed, 140 insertions(+), 70 deletions(-) diff --git a/bin/xbps-query/ownedby.c b/bin/xbps-query/ownedby.c index fe14965ea..2d8783a11 100644 --- a/bin/xbps-query/ownedby.c +++ b/bin/xbps-query/ownedby.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -42,35 +43,41 @@ #include -struct locate { - bool byhash; - const char* expr; - regex_t regex; +struct thread_data { + pthread_t this_thread; + struct archive* ar; + bool byhash; + const char* expr; + const regex_t* regex; + pthread_mutex_t* archive_mutex; + pthread_mutex_t* print_mutex; }; static inline int parse_line(char** fields, int fields_size, char* line) { - char* next; - int i = 0; + char* next; + int i = 0; - while (i < fields_size - 1) { - if (!(next = strchr(line, ':'))) - break; + while (i < fields_size - 1) { + if (!(next = strchr(line, '%'))) + break; - *next = '\0'; + *next = '\0'; - fields[i++] = *line ? line : NULL; - line = next + 1; - } + fields[i++] = *line ? line : NULL; + line = next + 1; + } fields[i++] = *line ? line : NULL; - return i; + return i; } -static inline void print_line(const char* pkg, char** file) { +static inline void print_line(pthread_mutex_t* mutex, const char* pkg, char** file) { + if (mutex) pthread_mutex_lock(mutex); printf("%s: %s", pkg, file[1]); if (file[2]) printf(" -> %s", file[2]); printf("\n"); + if (mutex) pthread_mutex_unlock(mutex); } static char* archive_get_file(struct archive* ar, struct archive_entry* entry) { @@ -81,10 +88,9 @@ static char* archive_get_file(struct archive* ar, struct archive_entry* entry) { assert(entry != NULL); buflen = archive_entry_size(entry); - buf = malloc(buflen + 1); + buf = malloc(buflen + 1); if (buf == NULL) return NULL; - if (archive_read_data(ar, buf, buflen) != buflen) { free(buf); return NULL; @@ -93,20 +99,68 @@ static char* archive_get_file(struct archive* ar, struct archive_entry* entry) { return buf; } -static int repo_search_files(struct xbps_repo* repo, void* locate_ptr, bool* done UNUSED) { - struct locate* locate = locate_ptr; - struct archive* ar; +static void* thread_func(void* data_ptr) { + struct thread_data* data = data_ptr; struct archive_entry* entry; - FILE* ar_file; - char* files_uri; - char* reponame_escaped; - if (!xbps_repository_is_remote(repo->uri)) { - files_uri = xbps_xasprintf("%s/%s-files", repo->uri, repo->xhp->target_arch ? repo->xhp->target_arch : repo->xhp->native_arch); + for (;;) { + char* pkg; + char* content; + char* line, *end_line; + char* file[3]; + int ar_rv; + + if (data->archive_mutex) pthread_mutex_lock(data->archive_mutex); + if ((ar_rv = archive_read_next_header(data->ar, &entry)) == ARCHIVE_OK) { + pkg = strdup(archive_entry_pathname(entry)); + content = archive_get_file(data->ar, entry); + } + if (data->archive_mutex) pthread_mutex_unlock(data->archive_mutex); + + if (ar_rv != ARCHIVE_OK) + break; + + line = strtok_r(content, "\n", &end_line); + while (line) { + parse_line(file, 3, line); + if (data->byhash) { + if (file[0] != NULL && strcasecmp(file[0], data->expr) == 0) + print_line(data->print_mutex, pkg, file); + } else if (data->expr != NULL) { + if (strcasestr(file[1], data->expr) != NULL || (file[2] && strcasestr(file[2], data->expr))) { + print_line(data->print_mutex, pkg, file); + } + } else { // regex + if (!regexec(data->regex, file[1], 0, NULL, 0) || + (file[2] && !regexec(data->regex, file[2], 0, NULL, 0))) { + print_line(data->print_mutex, pkg, file); + } + } + line = strtok_r(NULL, "\n", &end_line); + } + + free(pkg); + free(content); + } + + return NULL; +} + +static int repo_search_files(struct xbps_handle* xh, const char* repouri, bool byhash, const char* expr, regex_t* regex) { + struct archive* ar; + FILE* ar_file; + char* files_uri; + char* reponame_escaped; + pthread_mutex_t archive_mutex, print_mutex; + int maxthreads; + struct thread_data* thds; + + if (!xbps_repository_is_remote(repouri)) { + files_uri = xbps_xasprintf("%s/%s-files", repouri, xh->target_arch ? xh->target_arch : xh->native_arch); } else { - if (!(reponame_escaped = xbps_get_remote_repo_string(repo->uri))) + if (!(reponame_escaped = xbps_get_remote_repo_string(repouri))) return false; - files_uri = xbps_xasprintf("%s/%s/%s-files", repo->xhp->metadir, reponame_escaped, repo->xhp->target_arch ? repo->xhp->target_arch : repo->xhp->native_arch); + files_uri = xbps_xasprintf("%s/%s/%s-files", xh->metadir, reponame_escaped, xh->target_arch ? xh->target_arch : xh->native_arch); free(reponame_escaped); } @@ -125,39 +179,51 @@ static int repo_search_files(struct xbps_repo* repo, void* locate_ptr, bool* don archive_read_support_filter_zstd(ar); archive_read_support_format_tar(ar); - if (archive_read_open_FILE(ar, ar_file) == ARCHIVE_FATAL) { + if (archive_read_open_FILE(ar, ar_file) != ARCHIVE_OK) { xbps_dbg_printf("[repo] `%s' failed to open repodata archive %s\n", files_uri, archive_error_string(ar)); archive_read_close(ar); fclose(ar_file); return false; } - while (archive_read_next_header(ar, &entry) == ARCHIVE_OK) { - const char* pkg = archive_entry_pathname(entry); - char* content = archive_get_file(ar, entry); - char* line, *end_line; - char* file[3]; + maxthreads = (int) sysconf(_SC_NPROCESSORS_ONLN); + if (maxthreads <= 1) { + struct thread_data thd = { + .ar = ar, + .byhash = byhash, + .expr = expr, + .regex = regex, + .archive_mutex = NULL, + .print_mutex = NULL + }; + + thread_func(&thd); + } else { + thds = calloc(maxthreads, sizeof(*thds)); + assert(thds); - line = strtok_r(content, "\n", &end_line); - while (line) { - parse_line(file, 3, line); - if (locate->byhash) { - if (file[0] != NULL && strcasecmp(file[0], locate->expr) == 0) - print_line(pkg, file); - } else if (locate->expr != NULL) { - if (strcasestr(file[1], locate->expr) != NULL || (file[2] && strcasestr(file[2], locate->expr))) { - print_line(pkg, file); - } - } else { // regex - if (!regexec(&locate->regex, file[1], 0, NULL, 0) || - (file[2] && !regexec(&locate->regex, file[2], 0, NULL, 0))) { - print_line(pkg, file); - } + pthread_mutex_init(&archive_mutex, NULL); + pthread_mutex_init(&print_mutex, NULL); + + for (int i = 0; i < maxthreads; i++) { + int rv; + + thds[i].ar = ar; + thds[i].byhash = byhash; + thds[i].expr = expr; + thds[i].regex = regex; + thds[i].archive_mutex = &archive_mutex; + thds[i].print_mutex = &print_mutex; + + if ((rv = pthread_create(&thds[i].this_thread, NULL, thread_func, &thds[i])) != 0) { + xbps_error_printf("unable to create thread: %s, retrying...\n", strerror(rv)); + i--; } - line = strtok_r(NULL, "\n", &end_line); } - free(content); + for (int i = 0; i < maxthreads; i++) { + pthread_join(thds[i].this_thread, NULL); + } } fclose(ar_file); @@ -165,35 +231,35 @@ static int repo_search_files(struct xbps_repo* repo, void* locate_ptr, bool* don return 0; } -int ownedby(struct xbps_handle* xh, const char* file, bool repo_mode UNUSED, bool regex) { - struct locate locate; - - memset(&locate, 0, sizeof(locate)); +int ownedby(struct xbps_handle* xh, const char* file, bool repo_mode UNUSED, bool useregex) { + const char* expr = NULL; + regex_t regex; - if (regex) { - if (regcomp(&locate.regex, file, REG_EXTENDED | REG_NOSUB | REG_ICASE) != 0) { + if (useregex) { + if (regcomp(®ex, file, REG_EXTENDED | REG_NOSUB | REG_ICASE) != 0) { xbps_error_printf("xbps-locate: invalid regular expression\n"); exit(1); } } else { - locate.expr = file; + expr = file; } - xbps_rpool_foreach(xh, repo_search_files, &locate); + for (unsigned int i = 0; i < xbps_array_count(xh->repositories); i++) { + const char* repo_uri; + xbps_array_get_cstring_nocopy(xh->repositories, i, &repo_uri); + repo_search_files(xh, repo_uri, false, expr, ®ex); + } - if (regex) - regfree(&locate.regex); + if (useregex) + regfree(®ex); return 0; } int ownedhash(struct xbps_handle* xh, const char* file, bool repo_mode UNUSED, bool regex UNUSED) { - struct locate locate; - struct stat fstat; - char hash[XBPS_SHA256_SIZE]; - - memset(&locate, 0, sizeof(locate)); - locate.byhash = true; + const char* expr; + struct stat fstat; + char hash[XBPS_SHA256_SIZE]; if (stat(file, &fstat) != -1) { if (!S_ISREG(fstat.st_mode)) { @@ -204,12 +270,16 @@ int ownedhash(struct xbps_handle* xh, const char* file, bool repo_mode UNUSED, b xbps_error_printf("query: cannot get hash of `%s': %s\n", file, strerror(errno)); return 1; } - locate.expr = hash; + expr = hash; } else { - locate.expr = file; + expr = file; } - xbps_rpool_foreach(xh, repo_search_files, &locate); + for (unsigned int i = 0; i < xbps_array_count(xh->repositories); i++) { + const char* repo_uri; + xbps_array_get_cstring_nocopy(xh->repositories, i, &repo_uri); + repo_search_files(xh, repo_uri, true, expr, NULL); + } return 0; } diff --git a/bin/xbps-rindex/files-add.c b/bin/xbps-rindex/files-add.c index 0c0af2f82..0a76313e1 100644 --- a/bin/xbps-rindex/files-add.c +++ b/bin/xbps-rindex/files-add.c @@ -116,7 +116,7 @@ make_file_string(xbps_dictionary_t pkg, char* buffer, int buffer_size) { if (!xbps_dictionary_get_cstring_nocopy(pkg, "sha256", &pkg_sha256)) pkg_sha256 = ""; - if ((result = snprintf(buffer, buffer_size, "%s:%s:%s\n", pkg_sha256, pkg_file, pkg_target)) >= buffer_size) + if ((result = snprintf(buffer, buffer_size, "%s%%%s%%%s\n", pkg_sha256, pkg_file, pkg_target)) >= buffer_size) return errno = ENOBUFS, 0; if (result == -1) From d2db987ae086341e4f205b40c191ae850436ef38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Friedel=20Sch=C3=B6n?= Date: Wed, 31 Jan 2024 22:26:18 +0100 Subject: [PATCH 10/12] bin: rindex fixing copying from old-archive --- bin/xbps-rindex/files-add.c | 120 +++++++++++++++++++++--------------- 1 file changed, 69 insertions(+), 51 deletions(-) diff --git a/bin/xbps-rindex/files-add.c b/bin/xbps-rindex/files-add.c index 0a76313e1..418406a0c 100644 --- a/bin/xbps-rindex/files-add.c +++ b/bin/xbps-rindex/files-add.c @@ -42,6 +42,18 @@ #include #include +#define OPEN_AR_READ(ar) \ + { \ + (ar) = archive_read_new(); \ + assert(ar); \ + archive_read_support_filter_gzip(ar); \ + archive_read_support_filter_bzip2(ar); \ + archive_read_support_filter_xz(ar); \ + archive_read_support_filter_lz4(ar); \ + archive_read_support_filter_zstd(ar); \ + archive_read_support_format_tar(ar); \ + } + static void list_packages(xbps_array_t dest, struct archive* ar) { @@ -146,16 +158,17 @@ length_file_string(xbps_dictionary_t pkg) { } int files_add(struct xbps_handle* xhp, int args, int argmax, char** argv, bool force, const char* compression) { + static char pkg_line[4096]; xbps_dictionary_t props_plist, files_plist; xbps_array_t existing_files, ignore_packages; char * tmprepodir = NULL, *repodir = NULL, *new_ar_path, *rlockfname = NULL, *files_uri = NULL; int rv = 0, ret = 0, rlockfd = -1; - char pkg_line[65536]; FILE* old_ar_file; int new_ar_file; struct archive * old_ar = NULL, *new_ar; struct archive_entry* entry; mode_t mask; + int count_added = 0; existing_files = xbps_array_create(); ignore_packages = xbps_array_create(); @@ -183,15 +196,7 @@ int files_add(struct xbps_handle* xhp, int args, int argmax, char** argv, bool f goto out; } } else { - old_ar = archive_read_new(); - assert(old_ar); - - archive_read_support_filter_gzip(old_ar); - archive_read_support_filter_bzip2(old_ar); - archive_read_support_filter_xz(old_ar); - archive_read_support_filter_lz4(old_ar); - archive_read_support_filter_zstd(old_ar); - archive_read_support_format_tar(old_ar); + OPEN_AR_READ(old_ar); if (archive_read_open_FILE(old_ar, old_ar_file) == ARCHIVE_FATAL) { xbps_dbg_printf("[repo] `%s' failed to open repodata archive %s\n", files_uri, archive_error_string(old_ar)); @@ -199,8 +204,13 @@ int files_add(struct xbps_handle* xhp, int args, int argmax, char** argv, bool f } list_packages(existing_files, old_ar); - // maybe reopen? + archive_read_close(old_ar); fseek(old_ar_file, 0, SEEK_SET); + OPEN_AR_READ(old_ar); + if (archive_read_open_FILE(old_ar, old_ar_file) == ARCHIVE_FATAL) { + xbps_dbg_printf("[repo] `%s' failed to open repodata archive %s\n", files_uri, archive_error_string(old_ar)); + return false; + } } new_ar_path = xbps_xasprintf("%s.XXXXXXXXXX", files_uri); @@ -285,14 +295,13 @@ int files_add(struct xbps_handle* xhp, int args, int argmax, char** argv, bool f /* Same version or index version greater */ if (ret <= 0) { - fprintf(stderr, "files: skipping `%s' (%s), already registered.\n", pkgver, arch); xbps_object_release(props_plist); continue; } xbps_array_add_cstring(ignore_packages, dbpkgver); - printf("files: updating `%s' -> `%s'\n", dbpkgver, pkgver); + printf("files: updating `%s' -> `%s' (%s)\n", dbpkgver, pkgver, arch); } if ((files_plist = xbps_archive_fetch_plist(pkg, "/files.plist")) == NULL) { @@ -342,61 +351,70 @@ int files_add(struct xbps_handle* xhp, int args, int argmax, char** argv, bool f } archive_entry_free(file_entry); - printf("files: registered `%s'\n", pkgver); + count_added++; + printf("files: added `%s' (%s)\n", pkgver, arch); xbps_object_release(props_plist); xbps_object_release(files_plist); } - if (old_ar != NULL) { - while (archive_read_next_header(old_ar, &entry) == ARCHIVE_OK) { - char* buffer; - size_t buffer_size; + if (count_added > 0) { + if (old_ar != NULL) { + char buffer[1024]; + size_t buffer_size; + const char* pkgname; - if (xbps_match_string_in_array(ignore_packages, archive_entry_pathname(entry))) - continue; + while (archive_read_next_header(old_ar, &entry) == ARCHIVE_OK) { + pkgname = archive_entry_pathname(entry); + if (xbps_match_string_in_array(ignore_packages, pkgname)) { + archive_read_data_skip(old_ar); + continue; + } - if (archive_write_header(new_ar, entry) != ARCHIVE_OK) { - archive_entry_free(entry); - continue; - } + printf("files: copying `%s' from old archive\n", pkgname); - buffer_size = archive_entry_size(entry); - buffer = malloc(buffer_size); - archive_read_data(old_ar, buffer, buffer_size); + if (archive_write_header(new_ar, entry) != ARCHIVE_OK) { + archive_entry_free(entry); + continue; + } - if (archive_write_data(new_ar, buffer, buffer_size) != ARCHIVE_OK) { - archive_entry_free(entry); - free(buffer); - continue; - } + while ((buffer_size = archive_read_data(old_ar, buffer, sizeof(buffer))) > 0) { + assert(archive_write_data(new_ar, buffer, buffer_size) > 0); + } - if (archive_write_finish_entry(new_ar) != ARCHIVE_OK) { - archive_entry_free(entry); - free(buffer); - continue; + archive_write_finish_entry(new_ar); } - archive_entry_free(entry); + + archive_read_free(old_ar); } + /* Write data to tempfile and rename */ + if (archive_write_close(new_ar) != ARCHIVE_OK) + return false; + if (archive_write_free(new_ar) != ARCHIVE_OK) + return false; - archive_read_free(old_ar); - } + if (fchmod(new_ar_file, 0664) == -1) { + close(new_ar_file); + unlink(new_ar_path); + goto out; + } + close(new_ar_file); + if (rename(new_ar_path, files_uri) == -1) { + unlink(new_ar_path); + goto out; + } + } else { + if (old_ar != NULL) + archive_read_free(old_ar); - /* Write data to tempfile and rename */ - if (archive_write_close(new_ar) != ARCHIVE_OK) - return false; - if (archive_write_free(new_ar) != ARCHIVE_OK) - return false; + /* Write data to tempfile and rename */ + if (archive_write_close(new_ar) != ARCHIVE_OK) + return false; + if (archive_write_free(new_ar) != ARCHIVE_OK) + return false; - if (fchmod(new_ar_file, 0664) == -1) { close(new_ar_file); unlink(new_ar_path); - goto out; - } - close(new_ar_file); - if (rename(new_ar_path, files_uri) == -1) { - unlink(new_ar_path); - goto out; } out: From 16bfa49a0e92fd793ef9c1c5f2db0f6aba0d08e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Friedel=20Sch=C3=B6n?= Date: Wed, 31 Jan 2024 23:06:10 +0100 Subject: [PATCH 11/12] bin: rindex print both package-count after execute --- bin/xbps-rindex/defs.h | 4 ++-- bin/xbps-rindex/files-add.c | 13 +++++++++---- bin/xbps-rindex/index-add.c | 4 ++-- bin/xbps-rindex/main.c | 9 +++++++-- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/bin/xbps-rindex/defs.h b/bin/xbps-rindex/defs.h index 974bc2584..2de6b0086 100644 --- a/bin/xbps-rindex/defs.h +++ b/bin/xbps-rindex/defs.h @@ -31,10 +31,10 @@ #define _XBPS_RINDEX "xbps-rindex" /* From index-add.c */ -int index_add(struct xbps_handle *, int, int, char **, bool, const char *); +int index_add(struct xbps_handle *, int, int, char **, bool, const char *, int*); /* From files-add.c */ -int files_add(struct xbps_handle *, int, int, char **, bool, const char *); +int files_add(struct xbps_handle *, int, int, char **, bool, const char *, int*, int*); /* From index-clean.c */ int index_clean(struct xbps_handle *, const char *, bool, const char *); diff --git a/bin/xbps-rindex/files-add.c b/bin/xbps-rindex/files-add.c index 418406a0c..7f5b558d3 100644 --- a/bin/xbps-rindex/files-add.c +++ b/bin/xbps-rindex/files-add.c @@ -24,6 +24,7 @@ */ #include "defs.h" +#include "xbps/xbps_array.h" #include #include @@ -157,7 +158,7 @@ length_file_string(xbps_dictionary_t pkg) { return size; } -int files_add(struct xbps_handle* xhp, int args, int argmax, char** argv, bool force, const char* compression) { +int files_add(struct xbps_handle* xhp, int args, int argmax, char** argv, bool force, const char* compression, int* count_added, int* count_total) { static char pkg_line[4096]; xbps_dictionary_t props_plist, files_plist; xbps_array_t existing_files, ignore_packages; @@ -168,7 +169,8 @@ int files_add(struct xbps_handle* xhp, int args, int argmax, char** argv, bool f struct archive * old_ar = NULL, *new_ar; struct archive_entry* entry; mode_t mask; - int count_added = 0; + + *count_added = 0, *count_total = 0; existing_files = xbps_array_create(); ignore_packages = xbps_array_create(); @@ -204,6 +206,7 @@ int files_add(struct xbps_handle* xhp, int args, int argmax, char** argv, bool f } list_packages(existing_files, old_ar); + *count_total = xbps_array_count(existing_files); archive_read_close(old_ar); fseek(old_ar_file, 0, SEEK_SET); OPEN_AR_READ(old_ar); @@ -301,6 +304,7 @@ int files_add(struct xbps_handle* xhp, int args, int argmax, char** argv, bool f xbps_array_add_cstring(ignore_packages, dbpkgver); + (*count_total)--; printf("files: updating `%s' -> `%s' (%s)\n", dbpkgver, pkgver, arch); } @@ -351,14 +355,15 @@ int files_add(struct xbps_handle* xhp, int args, int argmax, char** argv, bool f } archive_entry_free(file_entry); - count_added++; + (*count_added)++; + (*count_total)++; printf("files: added `%s' (%s)\n", pkgver, arch); xbps_object_release(props_plist); xbps_object_release(files_plist); } - if (count_added > 0) { + if (*count_added > 0) { if (old_ar != NULL) { char buffer[1024]; size_t buffer_size; diff --git a/bin/xbps-rindex/index-add.c b/bin/xbps-rindex/index-add.c index 00b3132c9..fc3c25d40 100644 --- a/bin/xbps-rindex/index-add.c +++ b/bin/xbps-rindex/index-add.c @@ -200,7 +200,7 @@ repodata_commit(struct xbps_handle *xhp, const char *repodir, } int -index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force, const char *compression) +index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force, const char *compression, int *count) { xbps_dictionary_t idx, idxmeta, idxstage, binpkgd, curpkgd; struct xbps_repo *repo = NULL, *stage = NULL; @@ -383,7 +383,7 @@ index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force _XBPS_RINDEX, strerror(errno)); goto out; } - printf("index: %u packages registered.\n", xbps_dictionary_count(idx)); + *count = xbps_dictionary_count(idx); out: xbps_object_release(idx); diff --git a/bin/xbps-rindex/main.c b/bin/xbps-rindex/main.c index d1732685a..4a156d827 100644 --- a/bin/xbps-rindex/main.c +++ b/bin/xbps-rindex/main.c @@ -160,9 +160,14 @@ main(int argc, char **argv) } if (add_mode) { - rv = index_add(&xh, optind, argc, argv, force, compression); + int index_count = 0, files_count = 0, files_added = 0; + + rv = index_add(&xh, optind, argc, argv, force, compression, &index_count); if (rv == 0) - rv = files_add(&xh, optind, argc, argv, force, compression); + rv = files_add(&xh, optind, argc, argv, force, compression, &files_added, &files_count); + + printf("index: %d packages registered\n", index_count); + printf("files: %d packages registered, %d new packages\n", files_count, files_added); } else if (clean_mode) rv = index_clean(&xh, argv[optind], hashcheck, compression); else if (rm_mode) From 582bebdba715a40c89cd7f047d2ab97e602c5ac4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Friedel=20Sch=C3=B6n?= Date: Thu, 1 Feb 2024 17:35:49 +0100 Subject: [PATCH 12/12] bin: rindex/files update license name --- bin/xbps-rindex/files-add.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/xbps-rindex/files-add.c b/bin/xbps-rindex/files-add.c index 7f5b558d3..660aa809a 100644 --- a/bin/xbps-rindex/files-add.c +++ b/bin/xbps-rindex/files-add.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2012-2015 Juan Romero Pardines. + * Copyright (c) 2023 Friedel Schon. * All rights reserved. * * Redistribution and use in source and binary forms, with or without