Skip to content

Commit 6f5ece5

Browse files
atgreenatgreen
committed
Add support for OCI registries.
This patch introduces the ability to read repo contents from OCI registries, like ghcr.io, using the new 'oci' protocol. User can specify this protocol in their DNF .repo files as shown below: [oci-test] name=OCI Test baseurl=oci://ghcr.io/atgreen/librepo/gh-cli enabled=1 gpgcheck=1 To set up the server-side repository, create a public package repository in github, and populate it by pushing the repo file contents using the ORAS cli tool. createrepo . FILES=$(find . -type f | sed 's|^\./||') for FILE in $FILES; do oras push ghcr.io/atgreen/librepo/gh-cli/$FILE:latest $FILE done Currently, only public repositories are supported. To support private package repositories, a bearer token is required. Implementing this would necessitate changes to libdnf to allow for a bearer_token configuration option in .repo files. = changelog = msg: Add support for OCI registries type: enhancement
1 parent 3c85711 commit 6f5ece5

File tree

7 files changed

+173
-33
lines changed

7 files changed

+173
-33
lines changed

‎CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ FIND_PACKAGE(PkgConfig)
3535
PKG_CHECK_MODULES(GLIB2 glib-2.0>=2.66 gio-2.0 REQUIRED)
3636
PKG_SEARCH_MODULE(LIBCRYPTO REQUIRED libcrypto openssl)
3737
PKG_CHECK_MODULES(LIBXML2 libxml-2.0 REQUIRED)
38+
PKG_CHECK_MODULES(JSONGLIB json-glib-1.0 REQUIRED)
3839
FIND_PACKAGE(CURL 7.52.0 REQUIRED)
3940

4041
IF (USE_GPGME)
@@ -106,6 +107,7 @@ INCLUDE_DIRECTORIES("${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/l
106107

107108
INCLUDE_DIRECTORIES(${LIBXML2_INCLUDE_DIRS})
108109
INCLUDE_DIRECTORIES(${CURL_INCLUDE_DIR})
110+
INCLUDE_DIRECTORIES(${JSONGLIB_INCLUDE_DIRS})
109111
#INCLUDE_DIRECTORIES(${CHECK_INCLUDE_DIR})
110112
IF (USE_GPGME AND ENABLE_SELINUX)
111113
INCLUDE_DIRECTORIES(${SELINUX_INCLUDE_DIRS})

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ Fedora/Ubuntu name
1515
* glib2 (http://developer.gnome.org/glib/) - glib2-devel/libglib2.0-dev
1616
* libattr (https://savannah.nongnu.org/projects/attr) - libattr-devel/libattr1-dev
1717
* libcurl (http://curl.haxx.se/libcurl/) - libcurl-devel/libcurl4-openssl-dev
18+
* json-glib (https://wiki.gnome.org/Projects/JsonGlib) - json-glib/json-glib-devel
1819
* openssl (http://www.openssl.org/) - openssl-devel/libssl-dev
1920
* python (http://python.org/) - python3-devel/libpython3-dev
2021
* One of the libraries:

‎librepo.spec‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ BuildRequires: pkgconfig(rpm) >= 4.18.0
4848
%endif
4949
BuildRequires: libattr-devel
5050
BuildRequires: libcurl-devel >= %{libcurl_version}
51+
BuildRequires: pkgconfig(json-glib-1.0)
5152
BuildRequires: pkgconfig(libxml-2.0)
5253
BuildRequires: pkgconfig(libcrypto)
5354
%if %{need_selinux}

‎librepo/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ TARGET_LINK_LIBRARIES(librepo
6767
${LIBXML2_LIBRARIES}
6868
${CURL_LIBRARY}
6969
${LIBCRYPTO_LIBRARIES}
70+
${JSONGLIB_LIBRARIES}
7071
${GLIB2_LIBRARIES}
7172
)
7273
IF (USE_GPGME)

‎librepo/downloader.c‎

Lines changed: 164 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
#include <sys/xattr.h>
3636
#include <fcntl.h>
3737
#include <curl/curl.h>
38+
#include <json-glib/json-glib.h>
3839

3940
#ifdef WITH_ZCHUNK
4041
#include <zck.h>
@@ -85,6 +86,16 @@ typedef enum {
8586
All headers which we were looking for are already found*/
8687
} LrHeaderCbState;
8788

89+
/** Enum with OCI file status */
90+
typedef enum {
91+
LR_OCI_DL_WAITING, /*!<
92+
The OCI file is waiting to be processed. */
93+
LR_OCI_DL_MANIFEST, /*!<
94+
The OCI manifest file is being downloaded. */
95+
LR_OCI_DL_LAYER, /*!<
96+
The OCI layer is being downloaded. */
97+
} LrOciState;
98+
8899
/** Enum with zchunk file status */
89100
typedef enum {
90101
LR_ZCK_DL_HEADER_CK, /*!<
@@ -185,6 +196,8 @@ typedef struct {
185196
Last cb return code. */
186197
struct curl_slist *curl_rqheaders; /*!<
187198
Extra headers for request. */
199+
LrOciState oci_state; /*!<
200+
OCI download state. */
188201

189202
#ifdef WITH_ZCHUNK
190203
LrZckState zck_state; /*!<
@@ -1399,7 +1412,19 @@ open_target_file(LrTarget *target, GError **err)
13991412
int fd;
14001413
FILE *f;
14011414

1402-
if (target->target->fd != -1) {
1415+
if (target->oci_state == LR_OCI_DL_MANIFEST) {
1416+
if (target->target->fn == NULL) {
1417+
// Create a temporary file for the OCI manifest
1418+
const char *tmpdir = getenv("TMPDIR");
1419+
if (tmpdir == NULL)
1420+
tmpdir = "/tmp";
1421+
char *tmpname = g_strdup_printf("%s/librepo-oci-XXXXXX", tmpdir);
1422+
close (mkstemp(tmpname));
1423+
target->target->fn = tmpname;
1424+
}
1425+
}
1426+
1427+
if (target->oci_state != LR_OCI_DL_MANIFEST && target->target->fd != -1) {
14031428
// Use supplied filedescriptor
14041429
fd = dup(target->target->fd);
14051430
if (fd == -1) {
@@ -1435,6 +1460,80 @@ open_target_file(LrTarget *target, GError **err)
14351460
return f;
14361461
}
14371462

1463+
/** Get the OCI manifest URL
1464+
*/
1465+
static char *
1466+
get_oci_manifest_url(char *oci_url, GError **err)
1467+
{
1468+
assert(!err || *err == NULL);
1469+
1470+
// Remove the 'oci://' prefix
1471+
char *hostname = strdup(oci_url + 6);
1472+
1473+
char *first_slash = strchr(hostname, '/');
1474+
if (first_slash == NULL || first_slash[1] == 0) {
1475+
g_set_error(err, LR_DOWNLOADER_ERROR, LRE_IO,
1476+
"invalid OCI URL format: %s",
1477+
oci_url);
1478+
return NULL;
1479+
}
1480+
*first_slash = 0;
1481+
1482+
char *result = g_strdup_printf ("https://%s/v2/%s/manifests/latest",
1483+
hostname,
1484+
first_slash + 1);
1485+
free (hostname);
1486+
return result;
1487+
}
1488+
1489+
static char *
1490+
get_oci_layer_url(char *oci_url, char *fn, GError **err) {
1491+
assert(!err || *err == NULL);
1492+
1493+
// Load the JSON manifest file
1494+
JsonParser *parser = json_parser_new();
1495+
1496+
// Load the JSON manifest file
1497+
if (!json_parser_load_from_file(parser, fn, err)) {
1498+
g_object_unref(parser);
1499+
return NULL;
1500+
}
1501+
1502+
// Delete the manifest file
1503+
unlink(fn);
1504+
1505+
// Get the root object
1506+
JsonNode *root = json_parser_get_root(parser);
1507+
JsonObject *root_obj = json_node_get_object(root);
1508+
1509+
// Navigate to the layers array
1510+
JsonArray *layers = json_object_get_array_member(root_obj, "layers");
1511+
JsonObject *first_layer = json_array_get_object_element(layers, 0);
1512+
1513+
// Extract the digest for the first layer
1514+
const char *digest = json_object_get_string_member(first_layer, "digest");
1515+
1516+
// Remove the 'oci://' prefix
1517+
char *hostname = strdup(oci_url + 6);
1518+
1519+
char *first_slash = strchr(hostname, '/');
1520+
if (first_slash == NULL || first_slash[1] == 0) {
1521+
g_set_error(err, LR_DOWNLOADER_ERROR, LRE_IO,
1522+
"invalid OCI URL format: %s",
1523+
oci_url);
1524+
return NULL;
1525+
}
1526+
*first_slash = 0;
1527+
1528+
char *result = g_strdup_printf ("https://%s/v2/%s/blobs/%s",
1529+
hostname,
1530+
first_slash + 1,
1531+
digest);
1532+
free (hostname);
1533+
g_object_unref(parser);
1534+
return result;
1535+
}
1536+
14381537
/** Prepare next transfer
14391538
*/
14401539
static gboolean
@@ -1478,6 +1577,17 @@ prepare_next_transfer(LrDownload *dd, gboolean *candidatefound, GError **err)
14781577

14791578
protocol = lr_detect_protocol(full_url);
14801579

1580+
if (protocol == LR_PROTOCOL_OCI) {
1581+
if (target->oci_state == LR_OCI_DL_WAITING) {
1582+
target->oci_state = LR_OCI_DL_MANIFEST;
1583+
full_url = get_oci_manifest_url(full_url, err);
1584+
} else if (target->oci_state == LR_OCI_DL_LAYER) {
1585+
full_url = get_oci_layer_url(full_url, target->target->fn, err);
1586+
}
1587+
if (!full_url)
1588+
goto fail;
1589+
}
1590+
14811591
// Prepare CURL easy handle
14821592
CURLcode c_rc;
14831593
CURL *h;
@@ -1656,6 +1766,18 @@ prepare_next_transfer(LrDownload *dd, gboolean *candidatefound, GError **err)
16561766
if (!headers)
16571767
lr_out_of_memory();
16581768
}
1769+
if (target->oci_state == LR_OCI_DL_MANIFEST) {
1770+
headers = curl_slist_append(headers, "Authorization: Bearer QQ==");
1771+
if (!headers)
1772+
lr_out_of_memory();
1773+
headers = curl_slist_append(headers, "Accept: application/vnd.oci.image.manifest.v1+json");
1774+
if (!headers)
1775+
lr_out_of_memory();
1776+
} else if (target->oci_state == LR_OCI_DL_LAYER) {
1777+
headers = curl_slist_append(headers, "Authorization: Bearer QQ==");
1778+
if (!headers)
1779+
lr_out_of_memory();
1780+
}
16591781
target->curl_rqheaders = headers;
16601782
c_rc = curl_easy_setopt(h, CURLOPT_HTTPHEADER, headers);
16611783
assert(c_rc == CURLE_OK);
@@ -2324,15 +2446,17 @@ check_transfer_statuses(LrDownload *dd, GError **err)
23242446
// New file was downloaded - clear checksums cached in extended attributes
23252447
lr_checksum_clear_cache(fd);
23262448

2327-
ret = check_finished_transfer_checksum(fd,
2328-
target->target->checksums,
2329-
&matches,
2330-
&transfer_err,
2331-
&tmp_err);
2449+
ret = target->oci_state == LR_OCI_DL_WAITING
2450+
? check_finished_transfer_checksum(fd,
2451+
target->target->checksums,
2452+
&matches,
2453+
&transfer_err,
2454+
&tmp_err)
2455+
: 1;
23322456
if (!ret) { // Error
23332457
g_propagate_prefixed_error(err, tmp_err, "Downloading from %s"
2334-
"was successful but error encountered while "
2335-
"checksumming: ", effective_url);
2458+
"was successful but error encountered while "
2459+
"checksumming: ", effective_url);
23362460
return FALSE;
23372461
}
23382462
#ifdef WITH_ZCHUNK
@@ -2506,31 +2630,38 @@ check_transfer_statuses(LrDownload *dd, GError **err)
25062630
target->tried_mirrors = g_slist_remove(target->tried_mirrors, target->mirror);
25072631
} else {
25082632
#endif /* WITH_ZCHUNK */
2509-
target->state = LR_DS_FINISHED;
2510-
2511-
// Remove xattr that states that the file is being downloaded
2512-
// by librepo, because the file is now completely downloaded
2513-
// and the xattr is not needed (is is useful only for resuming)
2514-
remove_librepo_xattr(target->target);
2515-
2516-
// Call end callback
2517-
LrEndCb end_cb = target->target->endcb;
2518-
if (end_cb) {
2519-
int rc = end_cb(target->target->cbdata,
2520-
LR_TRANSFER_SUCCESSFUL,
2521-
NULL);
2522-
if (rc == LR_CB_ERROR) {
2523-
target->cb_return_code = LR_CB_ERROR;
2524-
g_debug("%s: Downloading was aborted by LR_CB_ERROR "
2525-
"from end callback", __func__);
2526-
g_set_error(&fail_fast_error, LR_DOWNLOADER_ERROR,
2527-
LRE_CBINTERRUPTED,
2528-
"Interrupted by LR_CB_ERROR from end callback");
2529-
}
2530-
}
2531-
if (target->mirror)
2532-
lr_downloadtarget_set_usedmirror(target->target,
2533-
target->mirror->mirror->url);
2633+
if (target->oci_state == LR_OCI_DL_MANIFEST) {
2634+
// Now that we have the manifest, let's download the
2635+
// layer blob.
2636+
target->oci_state = LR_OCI_DL_LAYER;
2637+
target->state = LR_DS_WAITING;
2638+
} else {
2639+
target->state = LR_DS_FINISHED;
2640+
2641+
// Remove xattr that states that the file is being downloaded
2642+
// by librepo, because the file is now completely downloaded
2643+
// and the xattr is not needed (is is useful only for resuming)
2644+
remove_librepo_xattr(target->target);
2645+
2646+
// Call end callback
2647+
LrEndCb end_cb = target->target->endcb;
2648+
if (end_cb) {
2649+
int rc = end_cb(target->target->cbdata,
2650+
LR_TRANSFER_SUCCESSFUL,
2651+
NULL);
2652+
if (rc == LR_CB_ERROR) {
2653+
target->cb_return_code = LR_CB_ERROR;
2654+
g_debug("%s: Downloading was aborted by LR_CB_ERROR "
2655+
"from end callback", __func__);
2656+
g_set_error(&fail_fast_error, LR_DOWNLOADER_ERROR,
2657+
LRE_CBINTERRUPTED,
2658+
"Interrupted by LR_CB_ERROR from end callback");
2659+
}
2660+
}
2661+
if (target->mirror)
2662+
lr_downloadtarget_set_usedmirror(target->target,
2663+
target->mirror->mirror->url);
2664+
}
25342665
#ifdef WITH_ZCHUNK
25352666
}
25362667
#endif /* WITH_ZCHUNK */

‎librepo/lrmirrorlist.c‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,9 @@ lr_detect_protocol(const char *url)
4242
if (g_str_has_prefix(url, "rsync://"))
4343
return LR_PROTOCOL_RSYNC;
4444

45+
if (g_str_has_prefix(url, "oci://"))
46+
return LR_PROTOCOL_OCI;
47+
4548
return LR_PROTOCOL_OTHER;
4649
}
4750

‎librepo/lrmirrorlist.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ typedef enum {
3535
LR_PROTOCOL_HTTP,
3636
LR_PROTOCOL_FTP,
3737
LR_PROTOCOL_RSYNC,
38+
LR_PROTOCOL_OCI,
3839
} LrProtocol;
3940

4041
/** A internal representation of a mirror */

0 commit comments

Comments
 (0)