pkg: clean support for repositories

Rework the way the bootstrap fetches pkg, by implementing a full support
for the repositories, the boostrap will now loop over all available repo
and try to fetch the full package from there. It will at the first valid
package found.

Fallback to packagesite (which has been deprecated for a while) if needed, by
transforming it into a repo, if no repo is found.

MFC After:	3 weeks

(cherry picked from commit dc4581589a3256667fafd46a30c67abdfd86618f)
This commit is contained in:
Baptiste Daroussin 2025-01-14 12:12:00 +01:00
parent 057409d025
commit 7be3ea2e64
3 changed files with 219 additions and 118 deletions

View file

@ -60,6 +60,8 @@ struct config_entry {
bool main_only; /* Only set in pkg.conf. */
};
static struct repositories repositories = STAILQ_HEAD_INITIALIZER(repositories);
static struct config_entry c[] = {
[PACKAGESITE] = {
PKG_CONFIG_STRING,
@ -212,7 +214,7 @@ boolstr_to_bool(const char *str)
}
static void
config_parse(const ucl_object_t *obj, pkg_conf_file_t conftype)
config_parse(const ucl_object_t *obj)
{
FILE *buffp;
char *buf = NULL;
@ -239,29 +241,9 @@ config_parse(const ucl_object_t *obj, pkg_conf_file_t conftype)
memset(buf, 0, bufsz);
rewind(buffp);
if (conftype == CONFFILE_PKG) {
for (j = 0; j < strlen(key); ++j)
fputc(toupper(key[j]), buffp);
fflush(buffp);
} else if (conftype == CONFFILE_REPO) {
if (strcasecmp(key, "url") == 0)
fputs("PACKAGESITE", buffp);
else if (strcasecmp(key, "mirror_type") == 0)
fputs("MIRROR_TYPE", buffp);
else if (strcasecmp(key, "signature_type") == 0)
fputs("SIGNATURE_TYPE", buffp);
else if (strcasecmp(key, "fingerprints") == 0)
fputs("FINGERPRINTS", buffp);
else if (strcasecmp(key, "pubkey") == 0)
fputs("PUBKEY", buffp);
else if (strcasecmp(key, "enabled") == 0) {
if ((cur->type != UCL_BOOLEAN) ||
!ucl_object_toboolean(cur))
goto cleanup;
} else
continue;
fflush(buffp);
}
for (i = 0; i < CONFIG_SIZE; i++) {
if (strcmp(buf, c[i].key) == 0)
@ -325,7 +307,7 @@ config_parse(const ucl_object_t *obj, pkg_conf_file_t conftype)
if (c[i].envset)
continue;
/* Prevent overriding ABI, ASSUME_ALWAYS_YES, etc. */
if (conftype != CONFFILE_PKG && c[i].main_only == true)
if (c[i].main_only == true)
continue;
switch (c[i].type) {
case PKG_CONFIG_LIST:
@ -337,12 +319,91 @@ config_parse(const ucl_object_t *obj, pkg_conf_file_t conftype)
}
}
cleanup:
free(temp_config);
fclose(buffp);
free(buf);
}
static void
parse_mirror_type(struct repository *r, const char *mt)
{
if (strcasecmp(mt, "srv") == 0)
r->mirror_type = MIRROR_SRV;
r->mirror_type = MIRROR_NONE;
}
static bool
parse_signature_type(struct repository *repo, const char *st)
{
if (strcasecmp(st, "FINGERPRINTS") == 0)
repo->signature_type = SIGNATURE_FINGERPRINT;
else if (strcasecmp(st, "PUBKEY") == 0)
repo->signature_type = SIGNATURE_PUBKEY;
else if (strcasecmp(st, "NONE") == 0)
repo->signature_type = SIGNATURE_NONE;
else {
warnx("Signature type %s is not supported for bootstraping,"
" ignoring repository %s", st, repo->name);
free(repo->url);
free(repo->name);
free(repo->fingerprints);
free(repo->pubkey);
free(repo);
return false;
}
return (true);
}
static void
parse_repo(const ucl_object_t *o)
{
const ucl_object_t *cur;
const char *key;
ucl_object_iter_t it = NULL;
struct repository *repo = calloc(1, sizeof(struct repository));
if (repo == NULL)
err(EXIT_FAILURE, "calloc");
repo->name = strdup(ucl_object_key(o));
if (repo->name == NULL)
err(EXIT_FAILURE, "strdup");
while ((cur = ucl_iterate_object(o, &it, true))) {
key = ucl_object_key(cur);
if (key == NULL)
continue;
if (strcasecmp(key, "url") == 0) {
repo->url = strdup(ucl_object_tostring(cur));
if (repo->url == NULL)
err(EXIT_FAILURE, "strdup");
} else if (strcasecmp(key, "mirror_type") == 0) {
parse_mirror_type(repo, ucl_object_tostring(cur));
} else if (strcasecmp(key, "signature_type") == 0) {
if (!parse_signature_type(repo, ucl_object_tostring(cur)))
return;
} else if (strcasecmp(key, "fingerprints") == 0) {
repo->fingerprints = strdup(ucl_object_tostring(cur));
if (repo->fingerprints == NULL)
err(EXIT_FAILURE, "strdup");
} else if (strcasecmp(key, "pubkey") == 0) {
repo->pubkey = strdup(ucl_object_tostring(cur));
if (repo->pubkey == NULL)
err(EXIT_FAILURE, "strdup");
} else if (strcasecmp(key, "enabled") == 0) {
if ((cur->type != UCL_BOOLEAN) ||
!ucl_object_toboolean(cur)) {
free(repo->url);
free(repo->name);
free(repo);
return;
}
}
}
STAILQ_INSERT_TAIL(&repositories, repo, next);
return;
}
/*-
* Parse new repo style configs in style:
* Name:
@ -368,8 +429,7 @@ parse_repo_file(ucl_object_t *obj, const char *requested_repo)
if (requested_repo != NULL && strcmp(requested_repo, key) != 0)
continue;
config_parse(cur, CONFFILE_REPO);
parse_repo(cur);
}
}
@ -380,8 +440,12 @@ read_conf_file(const char *confpath, const char *requested_repo,
{
struct ucl_parser *p;
ucl_object_t *obj = NULL;
const char *abi = pkg_get_myabi();
if (abi == NULL)
errx(EXIT_FAILURE, "Fail do determine ABI");
p = ucl_parser_new(0);
ucl_parser_register_variable(p, "ABI", abi);
if (!ucl_parser_add_file(p, confpath)) {
if (errno != ENOENT)
@ -398,7 +462,7 @@ read_conf_file(const char *confpath, const char *requested_repo,
"configuration file %s", confpath);
else {
if (conftype == CONFFILE_PKG)
config_parse(obj, conftype);
config_parse(obj);
else if (conftype == CONFFILE_REPO)
parse_repo_file(obj, requested_repo);
}
@ -409,7 +473,7 @@ read_conf_file(const char *confpath, const char *requested_repo,
return (0);
}
static int
static void
load_repositories(const char *repodir, const char *requested_repo)
{
struct dirent *ent;
@ -417,12 +481,9 @@ load_repositories(const char *repodir, const char *requested_repo)
char *p;
size_t n;
char path[MAXPATHLEN];
int ret;
ret = 0;
if ((d = opendir(repodir)) == NULL)
return (1);
return;
while ((ent = readdir(d))) {
/* Trim out 'repos'. */
@ -438,7 +499,6 @@ load_repositories(const char *repodir, const char *requested_repo)
continue;
if (read_conf_file(path, requested_repo,
CONFFILE_REPO)) {
ret = 1;
goto cleanup;
}
}
@ -446,8 +506,6 @@ load_repositories(const char *repodir, const char *requested_repo)
cleanup:
closedir(d);
return (ret);
}
int
@ -509,8 +567,7 @@ config_init(const char *requested_repo)
}
STAILQ_FOREACH(cv, c[REPOS_DIR].list, next)
if (load_repositories(cv->value, requested_repo))
goto finalize;
load_repositories(cv->value, requested_repo);
finalize:
if (c[ABI].val == NULL && c[ABI].value == NULL) {
@ -521,8 +578,6 @@ finalize:
c[ABI].val = abi;
}
subst_packagesite(c[ABI].value != NULL ? c[ABI].value : c[ABI].val);
return (0);
}
@ -561,6 +616,33 @@ config_bool(pkg_config_key k, bool *val)
return (0);
}
struct repositories *
config_get_repositories(void)
{
if (STAILQ_EMPTY(&repositories)) {
/* Fall back to PACKAGESITE - deprecated - */
struct repository *r = calloc(1, sizeof(r));
if (r == NULL)
err(EXIT_FAILURE, "calloc");
r->name = strdup("fallback");
if (r->name == NULL)
err(EXIT_FAILURE, "strdup");
subst_packagesite(c[ABI].value != NULL ? c[ABI].value : c[ABI].val);
r->url = c[PACKAGESITE].value;
if (c[SIGNATURE_TYPE].value != NULL)
if (!parse_signature_type(r, c[SIGNATURE_TYPE].value))
exit(EXIT_FAILURE);
if (c[MIRROR_TYPE].value != NULL)
parse_mirror_type(r, c[MIRROR_TYPE].value);
if (c[PUBKEY].value != NULL)
r->pubkey = c[PUBKEY].value;
if (c[FINGERPRINTS].value != NULL)
r->fingerprints = c[FINGERPRINTS].value;
STAILQ_INSERT_TAIL(&repositories, r, next);
}
return (&repositories);
}
void
config_finish(void) {
int i;

View file

@ -30,6 +30,7 @@
#define _PKG_CONFIG_H
#include <paths.h>
#include <sys/queue.h>
#define URL_SCHEME_PREFIX "pkg+"
@ -58,9 +59,32 @@ typedef enum {
CONFFILE_REPO,
} pkg_conf_file_t;
typedef enum {
SIGNATURE_NONE = 0,
SIGNATURE_FINGERPRINT,
SIGNATURE_PUBKEY,
} signature_t;
typedef enum {
MIRROR_NONE = 0,
MIRROR_SRV,
} mirror_t;
struct repository {
char *name;
char *url;
mirror_t mirror_type;
signature_t signature_type;
char *fingerprints;
char *pubkey;
STAILQ_ENTRY(repository) next;
};
STAILQ_HEAD(repositories, repository);
int config_init(const char *);
void config_finish(void);
int config_string(pkg_config_key, const char **);
int config_bool(pkg_config_key, bool *);
struct repositories *config_get_repositories(void);
#endif

View file

@ -256,7 +256,7 @@ install_pkg_static(const char *path, const char *pkgpath, bool force)
}
static int
fetch_to_fd(const char *url, char *path, const char *fetchOpts)
fetch_to_fd(struct repository *repo, const char *url, char *path, const char *fetchOpts)
{
struct url *u;
struct dns_srvinfo *mirrors, *current;
@ -268,18 +268,11 @@ fetch_to_fd(const char *url, char *path, const char *fetchOpts)
ssize_t r;
char buf[10240];
char zone[MAXHOSTNAMELEN + 13];
static const char *mirror_type = NULL;
max_retry = 3;
current = mirrors = NULL;
remote = NULL;
if (mirror_type == NULL && config_string(MIRROR_TYPE, &mirror_type)
!= 0) {
warnx("No MIRROR_TYPE defined");
return (-1);
}
if ((fd = mkstemp(path)) == -1) {
warn("mkstemp()");
return (-1);
@ -295,7 +288,7 @@ fetch_to_fd(const char *url, char *path, const char *fetchOpts)
while (remote == NULL) {
if (retry == max_retry) {
if (strcmp(u->scheme, "file") != 0 &&
strcasecmp(mirror_type, "srv") == 0) {
repo->mirror_type == MIRROR_SRV) {
snprintf(zone, sizeof(zone),
"_%s._tcp.%s", u->scheme, u->host);
mirrors = dns_getsrvinfo(zone);
@ -654,23 +647,31 @@ parse_cert(int fd) {
}
static bool
verify_pubsignature(int fd_pkg, int fd_sig)
verify_pubsignature(int fd_pkg, int fd_sig, struct repository *r)
{
struct pubkey *pk;
const char *pubkey;
char *data;
struct pkgsign_ctx *sctx;
size_t datasz;
bool ret;
const char *pubkey;
pk = NULL;
pubkey = NULL;
sctx = NULL;
data = NULL;
ret = false;
if (config_string(PUBKEY, &pubkey) != 0) {
warnx("No CONFIG_PUBKEY defined");
goto cleanup;
if (r != NULL) {
if (r->pubkey == NULL) {
warnx("No CONFIG_PUBKEY defined for %s", r->name);
goto cleanup;
}
pubkey = r->pubkey;
} else {
if (config_string(PUBKEY, &pubkey) != 0) {
warnx("No CONFIG_PUBKEY defined for %s", r->name);
goto cleanup;
}
}
if ((pk = read_pubkey(fd_sig)) == NULL) {
@ -704,8 +705,8 @@ verify_pubsignature(int fd_pkg, int fd_sig)
}
/* Verify the signature. */
printf("Verifying signature with public key %s... ", pubkey);
if (pkgsign_verify_data(sctx, data, datasz, pubkey, NULL, 0, pk->sig,
printf("Verifying signature with public key %s.a.. ", r->pubkey);
if (pkgsign_verify_data(sctx, data, datasz, r->pubkey, NULL, 0, pk->sig,
pk->siglen) == false) {
fprintf(stderr, "Signature is not valid\n");
goto cleanup;
@ -724,7 +725,7 @@ cleanup:
}
static bool
verify_signature(int fd_pkg, int fd_sig)
verify_signature(int fd_pkg, int fd_sig, struct repository *r)
{
struct fingerprint_list *trusted, *revoked;
struct fingerprint *fingerprint;
@ -743,9 +744,17 @@ verify_signature(int fd_pkg, int fd_sig)
ret = false;
/* Read and parse fingerprints. */
if (config_string(FINGERPRINTS, &fingerprints) != 0) {
warnx("No CONFIG_FINGERPRINTS defined");
goto cleanup;
if (r != NULL) {
if (r->fingerprints == NULL) {
warnx("No FINGERPRINTS defined for %s", r->name);
goto cleanup;
}
fingerprints = r->fingerprints;
} else {
if (config_string(FINGERPRINTS, &fingerprints) != 0) {
warnx("No FINGERPRINTS defined");
goto cleanup;
}
}
snprintf(path, MAXPATHLEN, "%s/trusted", fingerprints);
@ -834,7 +843,7 @@ cleanup:
}
static int
bootstrap_pkg(bool force, const char *fetchOpts)
bootstrap_pkg(bool force, const char *fetchOpts, struct repository *repo)
{
int fd_pkg, fd_sig;
int ret;
@ -842,28 +851,18 @@ bootstrap_pkg(bool force, const char *fetchOpts)
char tmppkg[MAXPATHLEN];
char tmpsig[MAXPATHLEN];
const char *packagesite;
const char *signature_type;
char pkgstatic[MAXPATHLEN];
const char *bootstrap_name;
fd_sig = -1;
ret = -1;
if (config_string(PACKAGESITE, &packagesite) != 0) {
warnx("No PACKAGESITE defined");
return (-1);
}
if (config_string(SIGNATURE_TYPE, &signature_type) != 0) {
warnx("Error looking up SIGNATURE_TYPE");
return (-1);
}
printf("Bootstrapping pkg from %s, please wait...\n", packagesite);
printf("Bootstrapping pkg from %s, please wait...\n", repo->url);
/* Support pkg+http:// for PACKAGESITE which is the new format
in 1.2 to avoid confusion on why http://pkg.FreeBSD.org has
no A record. */
packagesite = repo->url;
if (strncmp(URL_SCHEME_PREFIX, packagesite,
strlen(URL_SCHEME_PREFIX)) == 0)
packagesite += strlen(URL_SCHEME_PREFIX);
@ -874,53 +873,44 @@ bootstrap_pkg(bool force, const char *fetchOpts)
snprintf(tmppkg, MAXPATHLEN, "%s/%s.XXXXXX",
getenv("TMPDIR") ? getenv("TMPDIR") : _PATH_TMP,
bootstrap_name);
if ((fd_pkg = fetch_to_fd(url, tmppkg, fetchOpts)) != -1)
if ((fd_pkg = fetch_to_fd(repo, url, tmppkg, fetchOpts)) != -1)
break;
bootstrap_name = NULL;
}
if (bootstrap_name == NULL)
goto fetchfail;
if (signature_type != NULL &&
strcasecmp(signature_type, "NONE") != 0) {
if (strcasecmp(signature_type, "FINGERPRINTS") == 0) {
if (repo->signature_type == SIGNATURE_FINGERPRINT) {
snprintf(tmpsig, MAXPATHLEN, "%s/%s.sig.XXXXXX",
getenv("TMPDIR") ? getenv("TMPDIR") : _PATH_TMP,
bootstrap_name);
snprintf(url, MAXPATHLEN, "%s/Latest/%s.sig",
packagesite, bootstrap_name);
snprintf(tmpsig, MAXPATHLEN, "%s/%s.sig.XXXXXX",
getenv("TMPDIR") ? getenv("TMPDIR") : _PATH_TMP,
bootstrap_name);
snprintf(url, MAXPATHLEN, "%s/Latest/%s.sig",
packagesite, bootstrap_name);
if ((fd_sig = fetch_to_fd(url, tmpsig, fetchOpts)) == -1) {
fprintf(stderr, "Signature for pkg not "
"available.\n");
goto fetchfail;
}
if (verify_signature(fd_pkg, fd_sig) == false)
goto cleanup;
} else if (strcasecmp(signature_type, "PUBKEY") == 0) {
snprintf(tmpsig, MAXPATHLEN,
"%s/%s.pubkeysig.XXXXXX",
getenv("TMPDIR") ? getenv("TMPDIR") : _PATH_TMP,
bootstrap_name);
snprintf(url, MAXPATHLEN, "%s/Latest/%s.pubkeysig",
packagesite, bootstrap_name);
if ((fd_sig = fetch_to_fd(url, tmpsig, fetchOpts)) == -1) {
fprintf(stderr, "Signature for pkg not "
"available.\n");
goto fetchfail;
}
if (verify_pubsignature(fd_pkg, fd_sig) == false)
goto cleanup;
} else {
warnx("Signature type %s is not supported for "
"bootstrapping.", signature_type);
goto cleanup;
if ((fd_sig = fetch_to_fd(repo, url, tmpsig, fetchOpts)) == -1) {
fprintf(stderr, "Signature for pkg not "
"available.\n");
goto fetchfail;
}
if (verify_signature(fd_pkg, fd_sig, repo) == false)
goto cleanup;
} else if (repo->signature_type == SIGNATURE_PUBKEY) {
snprintf(tmpsig, MAXPATHLEN,
"%s/%s.pubkeysig.XXXXXX",
getenv("TMPDIR") ? getenv("TMPDIR") : _PATH_TMP,
bootstrap_name);
snprintf(url, MAXPATHLEN, "%s/Latest/%s.pubkeysig",
repo->url, bootstrap_name);
if ((fd_sig = fetch_to_fd(repo, url, tmpsig, fetchOpts)) == -1) {
fprintf(stderr, "Signature for pkg not "
"available.\n");
goto fetchfail;
}
if (verify_pubsignature(fd_pkg, fd_sig, repo) == false)
goto cleanup;
}
if ((ret = extract_pkg_static(fd_pkg, pkgstatic, MAXPATHLEN)) == 0)
@ -930,18 +920,15 @@ bootstrap_pkg(bool force, const char *fetchOpts)
fetchfail:
for (int j = 0; bootstrap_names[j] != NULL; j++) {
warnx("Attempted to fetch %s/Latest/%s", packagesite,
warnx("Attempted to fetch %s/Latest/%s", repo->url,
bootstrap_names[j]);
}
warnx("Error: %s", fetchLastErrString);
if (fetchLastErrCode == FETCH_RESOLV) {
fprintf(stderr, "Address resolution failed for %s.\n", packagesite);
fprintf(stderr, "Consider changing PACKAGESITE.\n");
} else {
fprintf(stderr, "A pre-built version of pkg could not be found for "
"your system.\n");
fprintf(stderr, "Consider changing PACKAGESITE or installing it from "
"ports: 'ports-mgmt/pkg'.\n");
}
cleanup:
@ -1025,7 +1012,7 @@ bootstrap_pkg_local(const char *pkgpath, bool force)
goto cleanup;
}
if (verify_signature(fd_pkg, fd_sig) == false)
if (verify_signature(fd_pkg, fd_sig, NULL) == false)
goto cleanup;
} else if (strcasecmp(signature_type, "PUBKEY") == 0) {
@ -1038,7 +1025,7 @@ bootstrap_pkg_local(const char *pkgpath, bool force)
goto cleanup;
}
if (verify_pubsignature(fd_pkg, fd_sig) == false)
if (verify_pubsignature(fd_pkg, fd_sig, NULL) == false)
goto cleanup;
} else {
@ -1106,6 +1093,7 @@ main(int argc, char *argv[])
signed char ch;
const char *fetchOpts;
char *command;
struct repositories *repositories;
activation_test = false;
add_pkg = false;
@ -1233,6 +1221,8 @@ main(int argc, char *argv[])
fetchDebug = 1;
if ((bootstrap_only && force) || access(pkgpath, X_OK) == -1) {
struct repository *repo;
int ret = 0;
/*
* To allow 'pkg -N' to be used as a reliable test for whether
* a system is configured to use pkg, don't bootstrap pkg
@ -1273,7 +1263,12 @@ main(int argc, char *argv[])
if (pkg_query_yes_no() == 0)
exit(EXIT_FAILURE);
}
if (bootstrap_pkg(force, fetchOpts) != 0)
repositories = config_get_repositories();
STAILQ_FOREACH(repo, repositories, next) {
if ((ret = bootstrap_pkg(force, fetchOpts, repo)) == 0)
break;
}
if (ret != 0)
exit(EXIT_FAILURE);
config_finish();