1 /*
2 * Copyright (C) 2018 Red Hat, Inc.
3 *
4 * Licensed under the GNU Lesser General Public License Version 2.1
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 */
20
21 #define METADATA_RELATIVE_DIR "repodata"
22 #define PACKAGES_RELATIVE_DIR "packages"
23 #define METALINK_FILENAME "metalink.xml"
24 #define MIRRORLIST_FILENAME "mirrorlist"
25 #define RECOGNIZED_CHKSUMS {"sha512", "sha256"}
26
27 #include "../log.hpp"
28 #include "Repo-private.hpp"
29 #include "../dnf-utils.h"
30 #include "../dnf-context.hpp"
31 #include "../hy-iutil.h"
32 #include "../hy-repo-private.hpp"
33 #include "../hy-util-private.hpp"
34 #include "../hy-iutil-private.hpp"
35 #include "../hy-types.h"
36 #include "libdnf/utils/File.hpp"
37 #include "libdnf/utils/utils.hpp"
38 #include "libdnf/utils/os-release.hpp"
39 #include "libdnf/utils/url-encode.hpp"
40
41 #include "bgettext/bgettext-lib.h"
42 #include "tinyformat/tinyformat.hpp"
43 #include <utils.hpp>
44
45 #include <librepo/librepo.h>
46 #include <fcntl.h>
47 #include <sys/stat.h>
48 #include <sys/types.h>
49 #include <unistd.h>
50 #include <utime.h>
51
52 #include <gpgme.h>
53
54 #include <solv/chksum.h>
55 #include <solv/repo.h>
56 #include <solv/util.h>
57
58 #include <array>
59 #include <atomic>
60 #include <cctype>
61 #include <cerrno>
62 #include <fstream>
63 #include <iomanip>
64 #include <iostream>
65 #include <list>
66 #include <map>
67 #include <set>
68 #include <sstream>
69 #include <system_error>
70 #include <type_traits>
71
72 #include <stdio.h>
73 #include <string.h>
74 #include <time.h>
75
76 #include <glib.h>
77
78 //
79 // COUNTME CONSTANTS
80 //
81 // width of the sliding time window (in seconds)
82 const int COUNTME_WINDOW = 7*24*60*60; // 1 week
83 // starting point of the sliding time window relative to the UNIX epoch
84 // allows for aligning the window with a specific weekday
85 const int COUNTME_OFFSET = 345600; // Monday (1970-01-05 00:00:00 UTC)
86 // estimated number of metalink requests sent over the window
87 // used to generate the probability distribution of counting events
88 const int COUNTME_BUDGET = 4; // metadata_expire defaults to 2 days
89 // cookie file name
90 const std::string COUNTME_COOKIE = "countme";
91 // cookie file format version
92 const int COUNTME_VERSION = 0;
93 // longevity buckets that we report in the flag
94 // example: {A, B, C} defines 4 buckets [0, A), [A, B), [B, C), [C, infinity)
95 // where each letter represents a window step (starting from 0)
96 const std::array<const int, 3> COUNTME_BUCKETS = { {2, 5, 25} };
97
98 namespace std {
99
100 template<>
101 struct default_delete<GError> {
operator ()std::default_delete102 void operator()(GError * ptr) noexcept { g_error_free(ptr); }
103 };
104
105 template<>
106 struct default_delete<LrResult> {
operator ()std::default_delete107 void operator()(LrResult * ptr) noexcept { lr_result_free(ptr); }
108 };
109
110 template<>
111 struct default_delete<LrPackageTarget> {
operator ()std::default_delete112 void operator()(LrPackageTarget * ptr) noexcept { lr_packagetarget_free(ptr); }
113 };
114
115 template<>
116 struct default_delete<std::remove_pointer<gpgme_ctx_t>::type> {
operator ()std::default_delete117 void operator()(gpgme_ctx_t ptr) noexcept { gpgme_release(ptr); }
118 };
119
120 } // namespace std
121
122 namespace libdnf {
123
124 class LrExceptionWithSourceUrl : public LrException {
125 public:
LrExceptionWithSourceUrl(int code,const std::string & msg,const std::string & sourceUrl)126 LrExceptionWithSourceUrl(int code, const std::string & msg, const std::string & sourceUrl)
127 : LrException(code, msg), sourceUrl(sourceUrl) {}
getSourceUrl() const128 const std::string & getSourceUrl() const { return sourceUrl; }
129 private:
130 std::string sourceUrl;
131 };
132
throwException(std::unique_ptr<GError> && err)133 static void throwException(std::unique_ptr<GError> && err)
134 {
135 throw LrException(err->code, err->message);
136 }
137
138 template<typename T>
handleSetOpt(LrHandle * handle,LrHandleOption option,T value)139 inline static void handleSetOpt(LrHandle * handle, LrHandleOption option, T value)
140 {
141 GError * errP{nullptr};
142 if (!lr_handle_setopt(handle, &errP, option, value)) {
143 throwException(std::unique_ptr<GError>(errP));
144 }
145 }
146
handleGetInfo(LrHandle * handle,LrHandleInfoOption option,void * value)147 inline static void handleGetInfo(LrHandle * handle, LrHandleInfoOption option, void * value)
148 {
149 GError * errP{nullptr};
150 if (!lr_handle_getinfo(handle, &errP, option, value)) {
151 throwException(std::unique_ptr<GError>(errP));
152 }
153 }
154
155 template<typename T>
resultGetInfo(LrResult * result,LrResultInfoOption option,T value)156 inline static void resultGetInfo(LrResult * result, LrResultInfoOption option, T value)
157 {
158 GError * errP{nullptr};
159 if (!lr_result_getinfo(result, &errP, option, value)) {
160 throwException(std::unique_ptr<GError>(errP));
161 }
162 }
163
164 /* Callback stuff */
165
progress(double totalToDownload,double downloaded)166 int RepoCB::progress(double totalToDownload, double downloaded) { return 0; }
fastestMirror(FastestMirrorStage stage,const char * ptr)167 void RepoCB::fastestMirror(FastestMirrorStage stage, const char * ptr) {}
handleMirrorFailure(const char * msg,const char * url,const char * metadata)168 int RepoCB::handleMirrorFailure(const char * msg, const char * url, const char * metadata) { return 0; }
169
repokeyImport(const std::string & id,const std::string & userId,const std::string & fingerprint,const std::string & url,long int timestamp)170 bool RepoCB::repokeyImport(const std::string & id, const std::string & userId,
171 const std::string & fingerprint, const std::string & url, long int timestamp)
172 {
173 return true;
174 }
175
176
177 // Map string from config option proxy_auth_method to librepo LrAuth value
178 static constexpr struct {
179 const char * name;
180 LrAuth code;
181 } PROXYAUTHMETHODS[] = {
182 {"none", LR_AUTH_NONE},
183 {"basic", LR_AUTH_BASIC},
184 {"digest", LR_AUTH_DIGEST},
185 {"negotiate", LR_AUTH_NEGOTIATE},
186 {"ntlm", LR_AUTH_NTLM},
187 {"digest_ie", LR_AUTH_DIGEST_IE},
188 {"ntlm_wb", LR_AUTH_NTLM_WB},
189 {"any", LR_AUTH_ANY}
190 };
191
endsWith(const std::string & str,const std::string & ending) const192 bool Repo::Impl::endsWith(const std::string &str, const std::string &ending) const {
193 if (str.length() >= ending.length())
194 return (str.compare(str.length() - ending.length(), ending.length(), ending) == 0);
195 else
196 return false;
197 }
198
getMetadataPath(const std::string & metadataType) const199 const std::string & Repo::Impl::getMetadataPath(const std::string &metadataType) const {
200 // auto logger(Log::getLogger());
201 static const std::string empty;
202 std::string lookupMetadataType = metadataType;
203 if (conf->getMainConfig().zchunk().getValue()) {
204 if(!endsWith(metadataType, "_zck"))
205 lookupMetadataType = metadataType + "_zck";
206 }
207 auto it = metadataPaths.find(lookupMetadataType);
208 if(it == metadataPaths.end() && lookupMetadataType != metadataType)
209 it = metadataPaths.find(metadataType);
210 auto & ret = (it != metadataPaths.end()) ? it->second : empty;
211 // if (ret.empty())
212 // logger->debug(tfm::format("not found \"%s\" for: %s", metadataType, conf->name().getValue()));
213 return ret;
214 }
215
progressCB(void * data,double totalToDownload,double downloaded)216 int Repo::Impl::progressCB(void * data, double totalToDownload, double downloaded)
217 {
218 if (!data)
219 return 0;
220 auto cbObject = static_cast<RepoCB *>(data);
221 return cbObject->progress(totalToDownload, downloaded);
222 }
223
fastestMirrorCB(void * data,LrFastestMirrorStages stage,void * ptr)224 void Repo::Impl::fastestMirrorCB(void * data, LrFastestMirrorStages stage, void *ptr)
225 {
226 if (!data)
227 return;
228 auto cbObject = static_cast<RepoCB *>(data);
229 const char * msg;
230 std::string msgString;
231 if (ptr) {
232 switch (stage) {
233 case LR_FMSTAGE_CACHELOADING:
234 case LR_FMSTAGE_CACHELOADINGSTATUS:
235 case LR_FMSTAGE_STATUS:
236 msg = static_cast<const char *>(ptr);
237 break;
238 case LR_FMSTAGE_DETECTION:
239 msgString = std::to_string(*((long *)ptr));
240 msg = msgString.c_str();
241 break;
242 default:
243 msg = nullptr;
244 }
245 } else
246 msg = nullptr;
247 cbObject->fastestMirror(static_cast<RepoCB::FastestMirrorStage>(stage), msg);
248 }
249
mirrorFailureCB(void * data,const char * msg,const char * url,const char * metadata)250 int Repo::Impl::mirrorFailureCB(void * data, const char * msg, const char * url, const char * metadata)
251 {
252 if (!data)
253 return 0;
254 auto cbObject = static_cast<RepoCB *>(data);
255 return cbObject->handleMirrorFailure(msg, url, metadata);
256 };
257
258
259 /**
260 * @brief Format user password string
261 *
262 * Returns user and password in user:password form. If quote is True,
263 * special characters in user and password are URL encoded.
264 *
265 * @param user Username
266 * @param passwd Password
267 * @param encode If quote is True, special characters in user and password are URL encoded.
268 * @return User and password in user:password form
269 */
formatUserPassString(const std::string & user,const std::string & passwd,bool encode)270 static std::string formatUserPassString(const std::string & user, const std::string & passwd, bool encode)
271 {
272 if (encode)
273 return urlEncode(user) + ":" + urlEncode(passwd);
274 else
275 return user + ":" + passwd;
276 }
277
Impl(Repo & owner,const std::string & id,Type type,std::unique_ptr<ConfigRepo> && conf)278 Repo::Impl::Impl(Repo & owner, const std::string & id, Type type, std::unique_ptr<ConfigRepo> && conf)
279 : id(id), type(type), conf(std::move(conf)), timestamp(-1), loadMetadataOther(false)
280 , syncStrategy(SyncStrategy::TRY_CACHE), owner(&owner), expired(false) {}
281
~Impl()282 Repo::Impl::~Impl()
283 {
284 g_strfreev(mirrors);
285 if (libsolvRepo)
286 libsolvRepo->appdata = nullptr;
287 }
288
Repo(const std::string & id,std::unique_ptr<ConfigRepo> && conf,Repo::Type type)289 Repo::Repo(const std::string & id, std::unique_ptr<ConfigRepo> && conf, Repo::Type type)
290 {
291 if (type == Type::AVAILABLE) {
292 auto idx = verifyId(id);
293 if (idx >= 0) {
294 std::string msg = tfm::format(
295 ("Invalid repository id \"%s\": invalid character '%s' at position %d."),
296 id, id[idx], idx + 1);
297 throw RepoError(msg);
298 }
299 }
300 pImpl.reset(new Impl(*this, id, type, std::move(conf)));
301 }
302
303 Repo::~Repo() = default;
304
setCallbacks(std::unique_ptr<RepoCB> && callbacks)305 void Repo::setCallbacks(std::unique_ptr<RepoCB> && callbacks)
306 {
307 pImpl->callbacks = std::move(callbacks);
308 }
309
verifyId(const std::string & id)310 int Repo::verifyId(const std::string & id)
311 {
312 auto idx = id.find_first_not_of(REPOID_CHARS);
313 return idx == id.npos ? -1 : idx;
314 }
315
verify() const316 void Repo::verify() const
317 {
318 if (pImpl->conf->baseurl().empty() &&
319 (pImpl->conf->metalink().empty() || pImpl->conf->metalink().getValue().empty()) &&
320 (pImpl->conf->mirrorlist().empty() || pImpl->conf->mirrorlist().getValue().empty()))
321 throw RepoError(tfm::format(_("Repository %s has no mirror or baseurl set."), pImpl->id));
322
323 const auto & type = pImpl->conf->type().getValue();
324 const char * supportedRepoTypes[]{"rpm-md", "rpm", "repomd", "rpmmd", "yum", "YUM"};
325 if (!type.empty()) {
326 for (auto supported : supportedRepoTypes) {
327 if (type == supported)
328 return;
329 }
330 throw RepoError(tfm::format(_("Repository '%s' has unsupported type: 'type=%s', skipping."),
331 pImpl->id, type));
332 }
333 }
334
getConfig()335 ConfigRepo * Repo::getConfig() noexcept
336 {
337 return pImpl->conf.get();
338 }
339
getId() const340 const std::string & Repo::getId() const noexcept
341 {
342 return pImpl->id;
343 }
344
enable()345 void Repo::enable()
346 {
347 pImpl->conf->enabled().set(Option::Priority::RUNTIME, true);
348 }
349
disable()350 void Repo::disable()
351 {
352 pImpl->conf->enabled().set(Option::Priority::RUNTIME, false);
353 }
354
isEnabled() const355 bool Repo::isEnabled() const
356 {
357 return pImpl->conf->enabled().getValue();
358 }
359
isLocal() const360 bool Repo::isLocal() const
361 {
362 auto & conf = pImpl->conf;
363 if ((!conf->metalink().empty() && !conf->metalink().getValue().empty()) ||
364 (!conf->mirrorlist().empty() && !conf->mirrorlist().getValue().empty()))
365 return false;
366 if (!conf->baseurl().getValue().empty() && conf->baseurl().getValue()[0].compare(0, 7, "file://") == 0)
367 return true;
368 return false;
369 }
370
getLocalBaseurl() const371 std::string Repo::getLocalBaseurl() const
372 {
373 if (!isLocal()) {
374 throw Exception("Invalid call getLocalBaseurl() on a non-local repository.");
375 }
376
377 // isLocal() already ensured the first item in the list starts with "file://"
378 return urlDecode(pImpl->conf->baseurl().getValue()[0].substr(7));
379 }
380
load()381 bool Repo::load() { return pImpl->load(); }
loadCache(bool throwExcept,bool ignoreMissing)382 bool Repo::loadCache(bool throwExcept, bool ignoreMissing) { return pImpl->loadCache(throwExcept, ignoreMissing); }
downloadMetadata(const std::string & destdir)383 void Repo::downloadMetadata(const std::string & destdir) { pImpl->downloadMetadata(destdir); }
getUseIncludes() const384 bool Repo::getUseIncludes() const { return pImpl->useIncludes; }
setUseIncludes(bool enabled)385 void Repo::setUseIncludes(bool enabled) { pImpl->useIncludes = enabled; }
getLoadMetadataOther() const386 bool Repo::getLoadMetadataOther() const { return pImpl->loadMetadataOther; }
setLoadMetadataOther(bool value)387 void Repo::setLoadMetadataOther(bool value) { pImpl->loadMetadataOther = value; }
getCost() const388 int Repo::getCost() const { return pImpl->conf->cost().getValue(); }
getPriority() const389 int Repo::getPriority() const { return pImpl->conf->priority().getValue(); }
getCompsFn()390 std::string Repo::getCompsFn() {
391 auto tmp = pImpl->getMetadataPath(MD_TYPE_GROUP_GZ);
392 if (tmp.empty())
393 tmp = pImpl->getMetadataPath(MD_TYPE_GROUP);
394 return tmp;
395 }
396
397
398 #ifdef MODULEMD
getModulesFn()399 std::string Repo::getModulesFn() { return pImpl->getMetadataPath(MD_TYPE_MODULES); }
400 #endif
401
getAge() const402 int Repo::getAge() const { return pImpl->getAge(); }
expire()403 void Repo::expire() { pImpl->expire(); }
isExpired() const404 bool Repo::isExpired() const { return pImpl->isExpired(); }
getExpiresIn() const405 int Repo::getExpiresIn() const { return pImpl->getExpiresIn(); }
406
setSubstitutions(const std::map<std::string,std::string> & substitutions)407 void Repo::setSubstitutions(const std::map<std::string, std::string> & substitutions)
408 {
409 pImpl->substitutions = substitutions;
410 }
411
addMetadataTypeToDownload(const std::string & metadataType)412 void Repo::addMetadataTypeToDownload(const std::string &metadataType)
413 {
414 pImpl->additionalMetadata.insert(metadataType);
415 }
416
removeMetadataTypeFromDownload(const std::string & metadataType)417 void Repo::removeMetadataTypeFromDownload(const std::string &metadataType)
418 {
419 pImpl->additionalMetadata.erase(metadataType);
420 }
421
getMetadataPath(const std::string & metadataType)422 std::string Repo::getMetadataPath(const std::string &metadataType)
423 {
424 return pImpl->getMetadataPath(metadataType);
425 }
426
getMetadataContent(const std::string & metadataType)427 std::string Repo::getMetadataContent(const std::string &metadataType)
428 {
429 auto path = getMetadataPath(metadataType);
430 if (path.empty()) return "";
431
432 auto mdfile = File::newFile(path);
433 mdfile->open("r");
434 const auto &content = mdfile->getContent();
435 mdfile->close();
436 return content;
437 }
438
lrHandleInitBase()439 std::unique_ptr<LrHandle> Repo::Impl::lrHandleInitBase()
440 {
441 std::unique_ptr<LrHandle> h(lr_handle_init());
442 std::vector<const char *> dlist = {MD_TYPE_PRIMARY, MD_TYPE_FILELISTS, MD_TYPE_PRESTODELTA,
443 MD_TYPE_GROUP_GZ, MD_TYPE_UPDATEINFO};
444
445 #ifdef MODULEMD
446 dlist.push_back(MD_TYPE_MODULES);
447 #endif
448 if (loadMetadataOther) {
449 dlist.push_back(MD_TYPE_OTHER);
450 }
451 for (auto &item : additionalMetadata) {
452 dlist.push_back(item.c_str());
453 }
454 dlist.push_back(NULL);
455 handleSetOpt(h.get(), LRO_PRESERVETIME, static_cast<long>(preserveRemoteTime));
456 handleSetOpt(h.get(), LRO_REPOTYPE, LR_YUMREPO);
457 handleSetOpt(h.get(), LRO_USERAGENT, conf->user_agent().getValue().c_str());
458 handleSetOpt(h.get(), LRO_YUMDLIST, dlist.data());
459 handleSetOpt(h.get(), LRO_INTERRUPTIBLE, 1L);
460 handleSetOpt(h.get(), LRO_GPGCHECK, conf->repo_gpgcheck().getValue());
461 handleSetOpt(h.get(), LRO_MAXMIRRORTRIES, static_cast<long>(maxMirrorTries));
462 handleSetOpt(h.get(), LRO_MAXPARALLELDOWNLOADS,
463 conf->max_parallel_downloads().getValue());
464
465 LrUrlVars * vars = NULL;
466 vars = lr_urlvars_set(vars, MD_TYPE_GROUP_GZ, MD_TYPE_GROUP);
467 handleSetOpt(h.get(), LRO_YUMSLIST, vars);
468
469 return h;
470 }
471
lrHandleInitLocal()472 std::unique_ptr<LrHandle> Repo::Impl::lrHandleInitLocal()
473 {
474 std::unique_ptr<LrHandle> h(lrHandleInitBase());
475
476 LrUrlVars * vars = NULL;
477 for (const auto & item : substitutions)
478 vars = lr_urlvars_set(vars, item.first.c_str(), item.second.c_str());
479 handleSetOpt(h.get(), LRO_VARSUB, vars);
480 auto cachedir = getCachedir();
481 handleSetOpt(h.get(), LRO_DESTDIR, cachedir.c_str());
482 const char *urls[] = {cachedir.c_str(), NULL};
483 handleSetOpt(h.get(), LRO_URLS, urls);
484 handleSetOpt(h.get(), LRO_LOCAL, 1L);
485 #ifdef LRO_SUPPORTS_CACHEDIR
486 /* If zchunk is enabled, set librepo cache dir */
487 if (conf->getMainConfig().zchunk().getValue()) {
488 if (conf->basecachedir().empty()) {
489 throw Exception(tfm::format(_("repo '%s': 'basecachedir' is not set"), id));
490 }
491 handleSetOpt(h.get(), LRO_CACHEDIR, conf->basecachedir().getValue().c_str());
492 }
493 #endif
494 return h;
495 }
496
497 template<typename ConfigT>
setHandle(LrHandle * h,ConfigT & config,const char * repoId=nullptr)498 static void setHandle(LrHandle * h, ConfigT & config, const char * repoId = nullptr) {
499 auto & ipResolve = config.ip_resolve().getValue();
500 if (ipResolve == "ipv4")
501 handleSetOpt(h, LRO_IPRESOLVE, LR_IPRESOLVE_V4);
502 else if (ipResolve == "ipv6")
503 handleSetOpt(h, LRO_IPRESOLVE, LR_IPRESOLVE_V6);
504
505 auto minrate = config.minrate().getValue();
506 handleSetOpt(h, LRO_LOWSPEEDLIMIT, static_cast<long>(minrate));
507
508 auto maxspeed = config.throttle().getValue();
509 if (maxspeed > 0 && maxspeed <= 1)
510 maxspeed *= config.bandwidth().getValue();
511 if (maxspeed != 0 && maxspeed < minrate)
512 throw RepoError(_("Maximum download speed is lower than minimum. "
513 "Please change configuration of minrate or throttle"));
514 handleSetOpt(h, LRO_MAXSPEED, static_cast<int64_t>(maxspeed));
515
516 long timeout = config.timeout().getValue();
517 if (timeout > 0) {
518 handleSetOpt(h, LRO_CONNECTTIMEOUT, timeout);
519 handleSetOpt(h, LRO_LOWSPEEDTIME, timeout);
520 } else {
521 handleSetOpt(h, LRO_CONNECTTIMEOUT, LRO_CONNECTTIMEOUT_DEFAULT);
522 handleSetOpt(h, LRO_LOWSPEEDTIME, LRO_LOWSPEEDTIME_DEFAULT);
523 }
524
525 // setup username/password if needed
526 auto userpwd = config.username().getValue();
527 if (!userpwd.empty()) {
528 // TODO Use URL encoded form, needs support in librepo
529 userpwd = formatUserPassString(userpwd, config.password().getValue(), false);
530 handleSetOpt(h, LRO_USERPWD, userpwd.c_str());
531 }
532
533 if (!config.proxy().empty() && !config.proxy().getValue().empty())
534 handleSetOpt(h, LRO_PROXY, config.proxy().getValue().c_str());
535
536 //set proxy authorization method
537 auto proxyAuthMethods = Repo::Impl::stringToProxyAuthMethods(config.proxy_auth_method().getValue());
538 handleSetOpt(h, LRO_PROXYAUTHMETHODS, static_cast<long>(proxyAuthMethods));
539
540 // setup proxy username/password if needed
541 if (!config.proxy_username().empty()) {
542 auto userpwd = config.proxy_username().getValue();
543 if (!userpwd.empty()) {
544 if (config.proxy_password().empty()) {
545 if (repoId)
546 throw RepoError(tfm::format(_("repo '%s': 'proxy_username' is set but not 'proxy_password'"), repoId));
547 else
548 throw RepoError(_("'proxy_username' is set but not 'proxy_password'"));
549 }
550 userpwd = formatUserPassString(userpwd, config.proxy_password().getValue(), true);
551 handleSetOpt(h, LRO_PROXYUSERPWD, userpwd.c_str());
552 }
553 }
554
555 // setup ssl stuff
556 if (!config.sslcacert().getValue().empty())
557 handleSetOpt(h, LRO_SSLCACERT, config.sslcacert().getValue().c_str());
558 if (!config.sslclientcert().getValue().empty())
559 handleSetOpt(h, LRO_SSLCLIENTCERT, config.sslclientcert().getValue().c_str());
560 if (!config.sslclientkey().getValue().empty())
561 handleSetOpt(h, LRO_SSLCLIENTKEY, config.sslclientkey().getValue().c_str());
562 long sslverify = config.sslverify().getValue() ? 1L : 0L;
563 handleSetOpt(h, LRO_SSLVERIFYHOST, sslverify);
564 handleSetOpt(h, LRO_SSLVERIFYPEER, sslverify);
565 handleSetOpt(h, LRO_SSLVERIFYSTATUS, config.sslverifystatus().getValue() ? 1L : 0L);
566
567 // setup proxy ssl stuff
568 if (!config.proxy_sslcacert().getValue().empty())
569 handleSetOpt(h, LRO_PROXY_SSLCACERT, config.proxy_sslcacert().getValue().c_str());
570 if (!config.proxy_sslclientcert().getValue().empty())
571 handleSetOpt(h, LRO_PROXY_SSLCLIENTCERT, config.proxy_sslclientcert().getValue().c_str());
572 if (!config.proxy_sslclientkey().getValue().empty())
573 handleSetOpt(h, LRO_PROXY_SSLCLIENTKEY, config.proxy_sslclientkey().getValue().c_str());
574 long proxy_sslverify = config.proxy_sslverify().getValue() ? 1L : 0L;
575 handleSetOpt(h, LRO_PROXY_SSLVERIFYHOST, proxy_sslverify);
576 handleSetOpt(h, LRO_PROXY_SSLVERIFYPEER, proxy_sslverify);
577 }
578
lrHandleInitRemote(const char * destdir)579 std::unique_ptr<LrHandle> Repo::Impl::lrHandleInitRemote(const char *destdir)
580 {
581 std::unique_ptr<LrHandle> h(lrHandleInitBase());
582 handleSetOpt(h.get(), LRO_HTTPHEADER, httpHeaders.get());
583
584 LrUrlVars * vars = NULL;
585 for (const auto & item : substitutions)
586 vars = lr_urlvars_set(vars, item.first.c_str(), item.second.c_str());
587 handleSetOpt(h.get(), LRO_VARSUB, vars);
588
589 handleSetOpt(h.get(), LRO_DESTDIR, destdir);
590
591 enum class Source {NONE, METALINK, MIRRORLIST} source{Source::NONE};
592 std::string tmp;
593 if (!conf->metalink().empty() && !(tmp=conf->metalink().getValue()).empty())
594 source = Source::METALINK;
595 else if (!conf->mirrorlist().empty() && !(tmp=conf->mirrorlist().getValue()).empty())
596 source = Source::MIRRORLIST;
597 if (source != Source::NONE) {
598 handleSetOpt(h.get(), LRO_PROGRESSDATA, callbacks.get());
599 if (source == Source::METALINK)
600 handleSetOpt(h.get(), LRO_METALINKURL, tmp.c_str());
601 else {
602 handleSetOpt(h.get(), LRO_MIRRORLISTURL, tmp.c_str());
603 // YUM-DNF compatibility hack. YUM guessed by content of keyword "metalink" if
604 // mirrorlist is really mirrorlist or metalink)
605 if (tmp.find("metalink") != tmp.npos)
606 handleSetOpt(h.get(), LRO_METALINKURL, tmp.c_str());
607 }
608 handleSetOpt(h.get(), LRO_FASTESTMIRROR, conf->fastestmirror().getValue() ? 1L : 0L);
609 if (conf->basecachedir().empty()) {
610 throw Exception(tfm::format(_("repo '%s': 'basecachedir' is not set"), id));
611 }
612 auto fastestMirrorCacheDir = conf->basecachedir().getValue();
613 if (fastestMirrorCacheDir.back() != '/')
614 fastestMirrorCacheDir.push_back('/');
615 fastestMirrorCacheDir += "fastestmirror.cache";
616 handleSetOpt(h.get(), LRO_FASTESTMIRRORCACHE, fastestMirrorCacheDir.c_str());
617 }
618
619 if (!conf->baseurl().getValue().empty()) {
620 size_t len = conf->baseurl().getValue().size();
621 const char * urls[len + 1];
622 for (size_t idx = 0; idx < len; ++idx)
623 urls[idx] = conf->baseurl().getValue()[idx].c_str();
624 urls[len] = nullptr;
625 handleSetOpt(h.get(), LRO_URLS, urls);
626 }
627
628 if (source == Source::NONE && conf->baseurl().getValue().empty())
629 throw RepoError(tfm::format(_("Cannot find a valid baseurl for repo: %s"), id));
630
631 handleSetOpt(h.get(), LRO_HMFCB, static_cast<LrHandleMirrorFailureCb>(mirrorFailureCB));
632 handleSetOpt(h.get(), LRO_PROGRESSCB, static_cast<LrProgressCb>(progressCB));
633 handleSetOpt(h.get(), LRO_PROGRESSDATA, callbacks.get());
634 handleSetOpt(h.get(), LRO_FASTESTMIRRORCB, static_cast<LrFastestMirrorCb>(fastestMirrorCB));
635 handleSetOpt(h.get(), LRO_FASTESTMIRRORDATA, callbacks.get());
636
637 #ifdef LRO_SUPPORTS_CACHEDIR
638 /* If zchunk is enabled, set librepo cache dir */
639 if (conf->getMainConfig().zchunk().getValue()) {
640 if (conf->basecachedir().empty()) {
641 throw Exception(tfm::format(_("repo '%s': 'basecachedir' is not set"), id));
642 }
643 handleSetOpt(h.get(), LRO_CACHEDIR, conf->basecachedir().getValue().c_str());
644 }
645 #endif
646
647 setHandle(h.get(), *conf, id.c_str());
648
649 return h;
650 }
651
gpgImportKey(gpgme_ctx_t context,int keyFd)652 static void gpgImportKey(gpgme_ctx_t context, int keyFd)
653 {
654 auto logger(Log::getLogger());
655 gpg_error_t gpgErr;
656 gpgme_data_t keyData;
657
658 gpgErr = gpgme_data_new_from_fd(&keyData, keyFd);
659 if (gpgErr != GPG_ERR_NO_ERROR) {
660 auto msg = tfm::format(_("%s: gpgme_data_new_from_fd(): %s"), __func__, gpgme_strerror(gpgErr));
661 logger->debug(msg);
662 throw LrException(LRE_GPGERROR, msg);
663 }
664
665 gpgErr = gpgme_op_import(context, keyData);
666 gpgme_data_release(keyData);
667 if (gpgErr != GPG_ERR_NO_ERROR) {
668 auto msg = tfm::format(_("%s: gpgme_op_import(): %s"), __func__, gpgme_strerror(gpgErr));
669 logger->debug(msg);
670 throw LrException(LRE_GPGERROR, msg);
671 }
672 }
673
gpgImportKey(gpgme_ctx_t context,std::vector<char> key)674 static void gpgImportKey(gpgme_ctx_t context, std::vector<char> key)
675 {
676 auto logger(Log::getLogger());
677 gpg_error_t gpgErr;
678 gpgme_data_t keyData;
679
680 gpgErr = gpgme_data_new_from_mem(&keyData, key.data(), key.size(), 0);
681 if (gpgErr != GPG_ERR_NO_ERROR) {
682 auto msg = tfm::format(_("%s: gpgme_data_new_from_fd(): %s"), __func__, gpgme_strerror(gpgErr));
683 logger->debug(msg);
684 throw LrException(LRE_GPGERROR, msg);
685 }
686
687 gpgErr = gpgme_op_import(context, keyData);
688 gpgme_data_release(keyData);
689 if (gpgErr != GPG_ERR_NO_ERROR) {
690 auto msg = tfm::format(_("%s: gpgme_op_import(): %s"), __func__, gpgme_strerror(gpgErr));
691 logger->debug(msg);
692 throw LrException(LRE_GPGERROR, msg);
693 }
694 }
695
rawkey2infos(int fd)696 static std::vector<Key> rawkey2infos(int fd) {
697 auto logger(Log::getLogger());
698 gpg_error_t gpgErr;
699
700 std::vector<Key> keyInfos;
701 gpgme_ctx_t ctx;
702 gpgme_new(&ctx);
703 std::unique_ptr<std::remove_pointer<gpgme_ctx_t>::type> context(ctx);
704
705 // set GPG home dir
706 char tmpdir[] = "/tmp/tmpdir.XXXXXX";
707 if (!mkdtemp(tmpdir)) {
708 const char * errTxt = strerror(errno);
709 throw RepoError(tfm::format(_("Cannot create repo temporary directory \"%s\": %s"),
710 tmpdir, errTxt));
711 }
712 Finalizer tmpDirRemover([&tmpdir](){
713 dnf_remove_recursive(tmpdir, NULL);
714 });
715 gpgErr = gpgme_ctx_set_engine_info(ctx, GPGME_PROTOCOL_OpenPGP, NULL, tmpdir);
716 if (gpgErr != GPG_ERR_NO_ERROR) {
717 auto msg = tfm::format(_("%s: gpgme_ctx_set_engine_info(): %s"), __func__, gpgme_strerror(gpgErr));
718 logger->debug(msg);
719 throw LrException(LRE_GPGERROR, msg);
720 }
721
722 gpgImportKey(ctx, fd);
723
724 gpgme_key_t key;
725 gpgErr = gpgme_op_keylist_start(ctx, NULL, 0);
726 while (!gpgErr)
727 {
728 gpgErr = gpgme_op_keylist_next(ctx, &key);
729 if (gpgErr)
730 break;
731
732 // _extract_signing_subkey
733 auto subkey = key->subkeys;
734 while (subkey && !subkey->can_sign) {
735 subkey = subkey->next;
736 }
737 if (subkey)
738 keyInfos.emplace_back(key, subkey);
739 gpgme_key_release(key);
740 }
741 if (gpg_err_code(gpgErr) != GPG_ERR_EOF)
742 {
743 gpgme_op_keylist_end(ctx);
744 auto msg = tfm::format(_("can not list keys: %s"), gpgme_strerror(gpgErr));
745 logger->debug(msg);
746 throw LrException(LRE_GPGERROR, msg);
747 }
748 gpgme_set_armor(ctx, 1);
749 for (auto & keyInfo : keyInfos) {
750 gpgme_data_t sink;
751 gpgme_data_new(&sink);
752 gpgme_op_export(ctx, keyInfo.getId().c_str(), 0, sink);
753 gpgme_data_rewind(sink);
754
755 char buf[4096];
756 ssize_t readed;
757 do {
758 readed = gpgme_data_read(sink, buf, sizeof(buf));
759 if (readed > 0)
760 keyInfo.rawKey.insert(keyInfo.rawKey.end(), buf, buf + sizeof(buf));
761 } while (readed == sizeof(buf));
762 }
763 return keyInfos;
764 }
765
keyidsFromPubring(const std::string & gpgDir)766 static std::vector<std::string> keyidsFromPubring(const std::string & gpgDir)
767 {
768 auto logger(Log::getLogger());
769 gpg_error_t gpgErr;
770
771 std::vector<std::string> keyids;
772
773 struct stat sb;
774 if (stat(gpgDir.c_str(), &sb) == 0 && S_ISDIR(sb.st_mode)) {
775
776 gpgme_ctx_t ctx;
777 gpgme_new(&ctx);
778 std::unique_ptr<std::remove_pointer<gpgme_ctx_t>::type> context(ctx);
779
780 // set GPG home dir
781 gpgErr = gpgme_ctx_set_engine_info(ctx, GPGME_PROTOCOL_OpenPGP, NULL, gpgDir.c_str());
782 if (gpgErr != GPG_ERR_NO_ERROR) {
783 auto msg = tfm::format(_("%s: gpgme_ctx_set_engine_info(): %s"), __func__, gpgme_strerror(gpgErr));
784 logger->debug(msg);
785 throw LrException(LRE_GPGERROR, msg);
786 }
787
788 gpgme_key_t key;
789 gpgErr = gpgme_op_keylist_start(ctx, NULL, 0);
790 while (!gpgErr)
791 {
792 gpgErr = gpgme_op_keylist_next(ctx, &key);
793 if (gpgErr)
794 break;
795
796 // _extract_signing_subkey
797 auto subkey = key->subkeys;
798 while (subkey && !key->subkeys->can_sign) {
799 subkey = subkey->next;
800 }
801 if (subkey)
802 keyids.push_back(subkey->keyid);
803 gpgme_key_release(key);
804 }
805 if (gpg_err_code(gpgErr) != GPG_ERR_EOF)
806 {
807 gpgme_op_keylist_end(ctx);
808 auto msg = tfm::format(_("can not list keys: %s"), gpgme_strerror(gpgErr));
809 logger->debug(msg);
810 throw LrException(LRE_GPGERROR, msg);
811 }
812 }
813 return keyids;
814 }
815
816 // download key from URL
retrieve(const std::string & url)817 std::vector<Key> Repo::Impl::retrieve(const std::string & url)
818 {
819 auto logger(Log::getLogger());
820 char tmpKeyFile[] = "/tmp/repokey.XXXXXX";
821 auto fd = mkstemp(tmpKeyFile);
822 if (fd == -1) {
823 auto msg = tfm::format("Error creating temporary file \"%s\": %s",
824 tmpKeyFile, std::system_category().message(errno));
825 logger->debug(msg);
826 throw LrException(LRE_GPGERROR, msg);
827 }
828 unlink(tmpKeyFile);
829 Finalizer tmpFileCloser([fd](){
830 close(fd);
831 });
832
833 try {
834 downloadUrl(url.c_str(), fd);
835 }
836 catch (const LrExceptionWithSourceUrl & e) {
837 auto msg = tfm::format(_("Failed to retrieve GPG key for repo '%s': %s"), id, e.what());
838 throw RepoError(msg);
839 }
840 lseek(fd, SEEK_SET, 0);
841 auto keyInfos = rawkey2infos(fd);
842 for (auto & key : keyInfos)
843 key.url = url;
844 return keyInfos;
845 }
846
847 /*
848 * Creates the '/run/user/$UID' directory if it doesn't exist. If this
849 * directory exists, gpgagent will create its sockets under
850 * '/run/user/$UID/gnupg'.
851 *
852 * If this directory doesn't exist, gpgagent will create its sockets in gpg
853 * home directory, which is under '/var/cache/yum/metadata/' and this was
854 * causing trouble with container images, see [1].
855 *
856 * Previous solution was to send the agent a "KILLAGENT" message, but that
857 * would cause a race condition with calling gpgme_release(), see [2], [3],
858 * [4].
859 *
860 * Since the agent doesn't clean up its sockets properly, by creating this
861 * directory we make sure they are in a place that is not causing trouble with
862 * container images.
863 *
864 * [1] https://bugzilla.redhat.com/show_bug.cgi?id=1650266
865 * [2] https://bugzilla.redhat.com/show_bug.cgi?id=1769831
866 * [3] https://github.com/rpm-software-management/microdnf/issues/50
867 * [4] https://bugzilla.redhat.com/show_bug.cgi?id=1781601
868 */
ensure_socket_dir_exists()869 static void ensure_socket_dir_exists() {
870 auto logger(Log::getLogger());
871 std::string dirname = "/run/user/" + std::to_string(getuid());
872 int res = mkdir(dirname.c_str(), 0700);
873 if (res != 0 && errno != EEXIST) {
874 logger->debug(tfm::format("Failed to create directory \"%s\": %d - %s",
875 dirname, errno, strerror(errno)));
876 }
877 }
878
importRepoKeys()879 void Repo::Impl::importRepoKeys()
880 {
881 auto logger(Log::getLogger());
882
883 auto gpgDir = getCachedir() + "/pubring";
884 auto knownKeys = keyidsFromPubring(gpgDir);
885 ensure_socket_dir_exists();
886 for (const auto & gpgkeyUrl : conf->gpgkey().getValue()) {
887 auto keyInfos = retrieve(gpgkeyUrl);
888 for (auto & keyInfo : keyInfos) {
889 if (std::find(knownKeys.begin(), knownKeys.end(), keyInfo.getId()) != knownKeys.end()) {
890 logger->debug(tfm::format(_("repo %s: 0x%s already imported"), id, keyInfo.getId()));
891 continue;
892 }
893
894 if (callbacks) {
895 if (!callbacks->repokeyImport(keyInfo.getId(), keyInfo.getUserId(), keyInfo.getFingerprint(),
896 keyInfo.url, keyInfo.getTimestamp()))
897 continue;
898 }
899
900 struct stat sb;
901 if (stat(gpgDir.c_str(), &sb) != 0 || !S_ISDIR(sb.st_mode)) {
902 int res = mkdir(gpgDir.c_str(), 0777);
903 if (res != 0 && errno != EEXIST) {
904 auto msg = tfm::format(_("Failed to create directory \"%s\": %d - %s"),
905 gpgDir, errno, strerror(errno));
906 throw RepoError(msg);
907 }
908 }
909
910 gpgme_ctx_t ctx;
911 gpgme_new(&ctx);
912 std::unique_ptr<std::remove_pointer<gpgme_ctx_t>::type> context(ctx);
913
914 // set GPG home dir
915 auto gpgErr = gpgme_ctx_set_engine_info(ctx, GPGME_PROTOCOL_OpenPGP, NULL, gpgDir.c_str());
916 if (gpgErr != GPG_ERR_NO_ERROR) {
917 auto msg = tfm::format(_("%s: gpgme_ctx_set_engine_info(): %s"), __func__, gpgme_strerror(gpgErr));
918 logger->debug(msg);
919 throw LrException(LRE_GPGERROR, msg);
920 }
921
922 gpgImportKey(ctx, keyInfo.rawKey);
923
924 logger->debug(tfm::format(_("repo %s: imported key 0x%s."), id, keyInfo.getId()));
925 }
926
927 }
928 }
929
lrHandlePerform(LrHandle * handle,const std::string & destDirectory,bool setGPGHomeDir)930 std::unique_ptr<LrResult> Repo::Impl::lrHandlePerform(LrHandle * handle, const std::string & destDirectory,
931 bool setGPGHomeDir)
932 {
933 if (setGPGHomeDir) {
934 auto pubringdir = getCachedir() + "/pubring";
935 handleSetOpt(handle, LRO_GNUPGHOMEDIR, pubringdir.c_str());
936 }
937
938 // Start and end is called only if progress callback is set in handle.
939 LrProgressCb progressFunc;
940 handleGetInfo(handle, LRI_PROGRESSCB, &progressFunc);
941
942 addCountmeFlag(handle);
943
944 std::unique_ptr<LrResult> result;
945 bool ret;
946 bool badGPG = false;
947 do {
948 if (callbacks && progressFunc)
949 callbacks->start(
950 !conf->name().getValue().empty() ? conf->name().getValue().c_str() :
951 (!id.empty() ? id.c_str() : "unknown")
952 );
953
954 GError * errP{nullptr};
955 result.reset(lr_result_init());
956 ret = lr_handle_perform(handle, result.get(), &errP);
957 std::unique_ptr<GError> err(errP);
958
959 if (callbacks && progressFunc)
960 callbacks->end();
961
962 if (ret || badGPG || errP->code != LRE_BADGPG) {
963 if (!ret) {
964 std::string source;
965 if (conf->metalink().empty() || (source=conf->metalink().getValue()).empty()) {
966 if (conf->mirrorlist().empty() || (source=conf->mirrorlist().getValue()).empty()) {
967 bool first = true;
968 for (const auto & url : conf->baseurl().getValue()) {
969 if (first)
970 first = false;
971 else
972 source += ", ";
973 source += url;
974 }
975 }
976 }
977 throw LrExceptionWithSourceUrl(err->code, err->message, source);
978 }
979 break;
980 }
981 badGPG = true;
982 importRepoKeys();
983 dnf_remove_recursive((destDirectory + "/" + METADATA_RELATIVE_DIR).c_str(), NULL);
984 } while (true);
985
986 return result;
987 }
988
loadCache(bool throwExcept,bool ignoreMissing)989 bool Repo::Impl::loadCache(bool throwExcept, bool ignoreMissing)
990 {
991 std::unique_ptr<LrHandle> h(lrHandleInitLocal());
992 std::unique_ptr<LrResult> r;
993
994 if (ignoreMissing) {
995 handleSetOpt(h.get(), LRO_IGNOREMISSING, 1L);
996 }
997
998 // Fetch data
999 try {
1000 r = lrHandlePerform(h.get(), getCachedir(), conf->repo_gpgcheck().getValue());
1001 } catch (std::exception & ex) {
1002 if (throwExcept)
1003 throw;
1004 return false;
1005 }
1006
1007 char **mirrors;
1008 LrYumRepo *yum_repo;
1009 LrYumRepoMd *yum_repomd;
1010 handleGetInfo(h.get(), LRI_MIRRORS, &mirrors);
1011 resultGetInfo(r.get(), LRR_YUM_REPO, &yum_repo);
1012 resultGetInfo(r.get(), LRR_YUM_REPOMD, &yum_repomd);
1013
1014 // Populate repo
1015 repomdFn = yum_repo->repomd;
1016 metadataPaths.clear();
1017 for (auto *elem = yum_repo->paths; elem; elem = g_slist_next(elem)) {
1018 if (elem->data) {
1019 auto yumrepopath = static_cast<LrYumRepoPath *>(elem->data);
1020 metadataPaths.emplace(yumrepopath->type, yumrepopath->path);
1021 }
1022 }
1023
1024 content_tags.clear();
1025 for (auto elem = yum_repomd->content_tags; elem; elem = g_slist_next(elem)) {
1026 if (elem->data)
1027 content_tags.emplace_back(static_cast<const char *>(elem->data));
1028 }
1029
1030 distro_tags.clear();
1031 for (auto elem = yum_repomd->distro_tags; elem; elem = g_slist_next(elem)) {
1032 if (elem->data) {
1033 auto distroTag = static_cast<LrYumDistroTag *>(elem->data);
1034 if (distroTag->tag)
1035 distro_tags.emplace_back(distroTag->cpeid, distroTag->tag);
1036 }
1037 }
1038
1039 metadata_locations.clear();
1040 for (auto elem = yum_repomd->records; elem; elem = g_slist_next(elem)) {
1041 if (elem->data) {
1042 auto rec = static_cast<LrYumRepoMdRecord *>(elem->data);
1043 metadata_locations.emplace_back(rec->type, rec->location_href);
1044 }
1045 }
1046
1047 if (auto cRevision = yum_repomd->revision) {
1048 revision = cRevision;
1049 }
1050 maxTimestamp = lr_yum_repomd_get_highest_timestamp(yum_repomd, NULL);
1051
1052 // Load timestamp unless explicitly expired
1053 if (timestamp != 0) {
1054 timestamp = mtime(getMetadataPath(MD_TYPE_PRIMARY).c_str());
1055 }
1056 g_strfreev(this->mirrors);
1057 this->mirrors = mirrors;
1058 return true;
1059 }
1060
addCountmeFlag(LrHandle * handle)1061 void Repo::Impl::addCountmeFlag(LrHandle *handle) {
1062 /*
1063 * The countme flag will be added once (and only once) in every position of
1064 * a sliding time window (COUNTME_WINDOW) that starts at COUNTME_OFFSET and
1065 * moves along the time axis, by one length at a time, in such a way that
1066 * the current point in time always stays within:
1067 *
1068 * UNIX epoch now
1069 * | |
1070 * |---*-----|-----|-----|-----[-*---]---> time
1071 * | ~~~~~~~
1072 * COUNTME_OFFSET COUNTME_WINDOW
1073 *
1074 * This is to align the time window with an absolute point in time rather
1075 * than the last counting event (which could facilitate tracking across
1076 * multiple such events).
1077 */
1078 auto logger(Log::getLogger());
1079
1080 // Bail out if not counting or not running as root (since the persistdir is
1081 // only root-writable)
1082 if (!conf->countme().getValue() || getuid() != 0)
1083 return;
1084
1085 // Bail out if not a remote handle
1086 long local;
1087 handleGetInfo(handle, LRI_LOCAL, &local);
1088 if (local)
1089 return;
1090
1091 // Bail out if no metalink or mirrorlist is defined
1092 auto & metalink = conf->metalink();
1093 auto & mirrorlist = conf->mirrorlist();
1094 if ((metalink.empty() || metalink.getValue().empty()) &&
1095 (mirrorlist.empty() || mirrorlist.getValue().empty()))
1096 return;
1097
1098 // Load the cookie
1099 std::string fname = getPersistdir() + "/" + COUNTME_COOKIE;
1100 int ver = COUNTME_VERSION; // file format version (for future use)
1101 time_t epoch = 0; // position of first-ever counted window
1102 time_t win = COUNTME_OFFSET; // position of last counted window
1103 int budget = -1; // budget for this window (-1 = generate)
1104 std::ifstream(fname) >> ver >> epoch >> win >> budget;
1105
1106 // Bail out if the window has not advanced since
1107 time_t now = time(NULL);
1108 time_t delta = now - win;
1109 if (delta < COUNTME_WINDOW) {
1110 logger->debug(tfm::format("countme: no event for %s: window already counted", id));
1111 return;
1112 }
1113
1114 // Evenly distribute the probability of the counting event over the first N
1115 // requests in this window (where N = COUNTME_BUDGET), by defining a random
1116 // "budget" of ordinary requests that we first have to spend. This ensures
1117 // that no particular request is special and thus no privacy loss is
1118 // incurred by adding the flag within N requests.
1119 if (budget < 0)
1120 budget = numeric::random(1, COUNTME_BUDGET);
1121 budget--;
1122 if (!budget) {
1123 // Budget exhausted, counting!
1124
1125 // Compute the position of this window
1126 win = now - (delta % COUNTME_WINDOW);
1127 if (!epoch)
1128 epoch = win;
1129 // Window step (0 at epoch)
1130 int step = (win - epoch) / COUNTME_WINDOW;
1131
1132 // Compute the bucket we are in
1133 unsigned int i;
1134 for (i = 0; i < COUNTME_BUCKETS.size(); ++i)
1135 if (step < COUNTME_BUCKETS[i])
1136 break;
1137 int bucket = i + 1; // Buckets are indexed from 1
1138
1139 // Set the flag
1140 std::string flag = "countme=" + std::to_string(bucket);
1141 handleSetOpt(handle, LRO_ONETIMEFLAG, flag.c_str());
1142 logger->debug(tfm::format("countme: event triggered for %s: bucket %i", id, bucket));
1143
1144 // Request a new budget
1145 budget = -1;
1146 } else {
1147 logger->debug(tfm::format("countme: no event for %s: budget to spend: %i", id, budget));
1148 }
1149
1150 // Save the cookie
1151 std::ofstream(fname) << COUNTME_VERSION << " " << epoch << " " << win
1152 << " " << budget;
1153 }
1154
1155 // Use metalink to check whether our metadata are still current.
isMetalinkInSync()1156 bool Repo::Impl::isMetalinkInSync()
1157 {
1158 auto logger(Log::getLogger());
1159 char tmpdir[] = "/tmp/tmpdir.XXXXXX";
1160 if (!mkdtemp(tmpdir)) {
1161 const char * errTxt = strerror(errno);
1162 throw RepoError(tfm::format(_("Cannot create repo temporary directory \"%s\": %s"),
1163 tmpdir, errTxt));
1164 }
1165 Finalizer tmpDirRemover([&tmpdir](){
1166 dnf_remove_recursive(tmpdir, NULL);
1167 });
1168
1169 std::unique_ptr<LrHandle> h(lrHandleInitRemote(tmpdir));
1170
1171 handleSetOpt(h.get(), LRO_FETCHMIRRORS, 1L);
1172 auto r = lrHandlePerform(h.get(), tmpdir, false);
1173 LrMetalink * metalink;
1174 handleGetInfo(h.get(), LRI_METALINK, &metalink);
1175 if (!metalink) {
1176 logger->debug(tfm::format(_("reviving: repo '%s' skipped, no metalink."), id));
1177 return false;
1178 }
1179
1180 // check all recognized hashes
1181 auto chksumFree = [](Chksum * ptr){solv_chksum_free(ptr, nullptr);};
1182 struct hashInfo {
1183 const LrMetalinkHash * lrMetalinkHash;
1184 std::unique_ptr<Chksum, decltype(chksumFree)> chksum;
1185 };
1186 std::vector<hashInfo> hashes;
1187 for (auto hash = metalink->hashes; hash; hash = hash->next) {
1188 auto lrMetalinkHash = static_cast<const LrMetalinkHash *>(hash->data);
1189 for (auto algorithm : RECOGNIZED_CHKSUMS) {
1190 if (strcmp(lrMetalinkHash->type, algorithm) == 0)
1191 hashes.push_back({lrMetalinkHash, {nullptr, chksumFree}});
1192 }
1193 }
1194 if (hashes.empty()) {
1195 logger->debug(tfm::format(_("reviving: repo '%s' skipped, no usable hash."), id));
1196 return false;
1197 }
1198
1199 for (auto & hash : hashes) {
1200 auto chkType = solv_chksum_str2type(hash.lrMetalinkHash->type);
1201 hash.chksum.reset(solv_chksum_create(chkType));
1202 }
1203
1204 std::ifstream repomd(repomdFn, std::ifstream::binary);
1205 char buf[4096];
1206 int readed;
1207 while ((readed = repomd.readsome(buf, sizeof(buf))) > 0) {
1208 for (auto & hash : hashes)
1209 solv_chksum_add(hash.chksum.get(), buf, readed);
1210 }
1211
1212 for (auto & hash : hashes) {
1213 int chksumLen;
1214 auto chksum = solv_chksum_get(hash.chksum.get(), &chksumLen);
1215 char chksumHex[chksumLen * 2 + 1];
1216 solv_bin2hex(chksum, chksumLen, chksumHex);
1217 if (strcmp(chksumHex, hash.lrMetalinkHash->value) != 0) {
1218 logger->debug(tfm::format(_("reviving: failed for '%s', mismatched %s sum."),
1219 id, hash.lrMetalinkHash->type));
1220 return false;
1221 }
1222 }
1223
1224 logger->debug(tfm::format(_("reviving: '%s' can be revived - metalink checksums match."), id));
1225 return true;
1226 }
1227
1228 // Use repomd to check whether our metadata are still current.
isRepomdInSync()1229 bool Repo::Impl::isRepomdInSync()
1230 {
1231 auto logger(Log::getLogger());
1232 LrYumRepo *yum_repo;
1233 char tmpdir[] = "/tmp/tmpdir.XXXXXX";
1234 if (!mkdtemp(tmpdir)) {
1235 const char * errTxt = strerror(errno);
1236 throw RepoError(tfm::format(_("Cannot create repo temporary directory \"%s\": %s"),
1237 tmpdir, errTxt));
1238 }
1239 Finalizer tmpDirRemover([&tmpdir](){
1240 dnf_remove_recursive(tmpdir, NULL);
1241 });
1242
1243 const char *dlist[] = LR_YUM_REPOMDONLY;
1244
1245 std::unique_ptr<LrHandle> h(lrHandleInitRemote(tmpdir));
1246
1247 handleSetOpt(h.get(), LRO_YUMDLIST, dlist);
1248 auto r = lrHandlePerform(h.get(), tmpdir, conf->repo_gpgcheck().getValue());
1249 resultGetInfo(r.get(), LRR_YUM_REPO, &yum_repo);
1250
1251 auto same = haveFilesSameContent(repomdFn.c_str(), yum_repo->repomd);
1252 if (same)
1253 logger->debug(tfm::format(_("reviving: '%s' can be revived - repomd matches."), id));
1254 else
1255 logger->debug(tfm::format(_("reviving: failed for '%s', mismatched repomd."), id));
1256 return same;
1257 }
1258
isInSync()1259 bool Repo::Impl::isInSync()
1260 {
1261 if (!conf->metalink().empty() && !conf->metalink().getValue().empty())
1262 return isMetalinkInSync();
1263 return isRepomdInSync();
1264 }
1265
1266
1267
fetch(const std::string & destdir,std::unique_ptr<LrHandle> && h)1268 void Repo::Impl::fetch(const std::string & destdir, std::unique_ptr<LrHandle> && h)
1269 {
1270 auto repodir = destdir + "/" + METADATA_RELATIVE_DIR;
1271 if (g_mkdir_with_parents(destdir.c_str(), 0755) == -1) {
1272 const char * errTxt = strerror(errno);
1273 throw RepoError(tfm::format(_("Cannot create repo destination directory \"%s\": %s"),
1274 destdir, errTxt));
1275 }
1276 auto tmpdir = destdir + "/tmpdir.XXXXXX";
1277 if (!mkdtemp(&tmpdir.front())) {
1278 const char * errTxt = strerror(errno);
1279 throw RepoError(tfm::format(_("Cannot create repo temporary directory \"%s\": %s"),
1280 tmpdir.c_str(), errTxt));
1281 }
1282 Finalizer tmpDirRemover([&tmpdir](){
1283 dnf_remove_recursive(tmpdir.c_str(), NULL);
1284 });
1285 auto tmprepodir = tmpdir + "/" + METADATA_RELATIVE_DIR;
1286
1287 handleSetOpt(h.get(), LRO_DESTDIR, tmpdir.c_str());
1288 auto r = lrHandlePerform(h.get(), tmpdir, conf->repo_gpgcheck().getValue());
1289
1290 dnf_remove_recursive(repodir.c_str(), NULL);
1291 if (g_mkdir_with_parents(repodir.c_str(), 0755) == -1) {
1292 const char * errTxt = strerror(errno);
1293 throw RepoError(tfm::format(_("Cannot create directory \"%s\": %s"),
1294 repodir, errTxt));
1295 }
1296 // move all downloaded object from tmpdir to destdir
1297 if (auto * dir = opendir(tmpdir.c_str())) {
1298 Finalizer tmpDirRemover([dir](){ closedir(dir); });
1299 while (auto ent = readdir(dir)) {
1300 auto elName = ent->d_name;
1301 if (elName[0] == '.' && (elName[1] == '\0' || (elName[1] == '.' && elName[2] == '\0'))) {
1302 continue;
1303 }
1304 auto targetElement = destdir + "/" + elName;
1305 if (filesystem::exists(targetElement)) {
1306 if (filesystem::isDIR(targetElement.c_str())) {
1307 dnf_remove_recursive(targetElement.c_str(), NULL);
1308 } else {
1309 dnf_ensure_file_unlinked(targetElement.c_str(), NULL);
1310 }
1311 }
1312 auto tempElement = tmpdir + "/" + elName;
1313 GError * error = NULL;
1314 if (!dnf_move_recursive(tempElement.c_str(), targetElement.c_str(), &error)) {
1315 std::string errTxt = tfm::format(
1316 _("Cannot rename directory \"%s\" to \"%s\": %s"),
1317 tempElement, targetElement, error->message);
1318 g_error_free(error);
1319 throw RepoError(errTxt);
1320 }
1321 }
1322 }
1323 }
1324
downloadMetadata(const std::string & destdir)1325 void Repo::Impl::downloadMetadata(const std::string & destdir)
1326 {
1327 std::unique_ptr<LrHandle> h(lrHandleInitRemote(nullptr));
1328 handleSetOpt(h.get(), LRO_YUMDLIST, LR_RPMMD_FULL);
1329 fetch(destdir, std::move(h));
1330 }
1331
load()1332 bool Repo::Impl::load()
1333 {
1334 auto logger(Log::getLogger());
1335 try {
1336 if (!getMetadataPath(MD_TYPE_PRIMARY).empty() || loadCache(false)) {
1337 resetMetadataExpired();
1338 if (!expired || syncStrategy == SyncStrategy::ONLY_CACHE || syncStrategy == SyncStrategy::LAZY) {
1339 logger->debug(tfm::format(_("repo: using cache for: %s"), id));
1340 return false;
1341 }
1342
1343 if (isInSync()) {
1344 // the expired metadata still reflect the origin:
1345 utimes(getMetadataPath(MD_TYPE_PRIMARY).c_str(), NULL);
1346 expired = false;
1347 return true;
1348 }
1349 }
1350 if (syncStrategy == SyncStrategy::ONLY_CACHE) {
1351 auto msg = tfm::format(_("Cache-only enabled but no cache for '%s'"), id);
1352 throw RepoError(msg);
1353 }
1354
1355 logger->debug(tfm::format(_("repo: downloading from remote: %s"), id));
1356 const auto cacheDir = getCachedir();
1357 fetch(cacheDir, lrHandleInitRemote(nullptr));
1358 timestamp = -1;
1359 loadCache(true);
1360 fresh = true;
1361 } catch (const LrExceptionWithSourceUrl & e) {
1362 auto msg = tfm::format(_("Failed to download metadata for repo '%s': %s"), id, e.what());
1363 throw RepoError(msg);
1364 }
1365 expired = false;
1366 return true;
1367 }
1368
getHash() const1369 std::string Repo::Impl::getHash() const
1370 {
1371 std::string tmp;
1372 if (conf->metalink().empty() || (tmp=conf->metalink().getValue()).empty()) {
1373 if (conf->mirrorlist().empty() || (tmp=conf->mirrorlist().getValue()).empty()) {
1374 if (!conf->baseurl().getValue().empty())
1375 tmp = conf->baseurl().getValue()[0];
1376 if (tmp.empty())
1377 tmp = id;
1378 }
1379 }
1380
1381 auto chksumObj = solv_chksum_create(REPOKEY_TYPE_SHA256);
1382 solv_chksum_add(chksumObj, tmp.c_str(), tmp.length());
1383 int chksumLen;
1384 auto chksum = solv_chksum_get(chksumObj, &chksumLen);
1385 static constexpr int USE_CHECKSUM_BYTES = 8;
1386 if (chksumLen < USE_CHECKSUM_BYTES) {
1387 solv_chksum_free(chksumObj, nullptr);
1388 throw Exception(_("getCachedir(): Computation of SHA256 failed"));
1389 }
1390 char chksumCStr[USE_CHECKSUM_BYTES * 2 + 1];
1391 solv_bin2hex(chksum, USE_CHECKSUM_BYTES, chksumCStr);
1392 solv_chksum_free(chksumObj, nullptr);
1393
1394 return id + "-" + chksumCStr;
1395 }
1396
getCachedir() const1397 std::string Repo::Impl::getCachedir() const
1398 {
1399 if (conf->basecachedir().empty()) {
1400 throw Exception(tfm::format(_("repo '%s': 'basecachedir' is not set"), id));
1401 }
1402 auto repodir(conf->basecachedir().getValue());
1403 if (repodir.back() != '/')
1404 repodir.push_back('/');
1405 return repodir + getHash();
1406 }
1407
getPersistdir() const1408 std::string Repo::Impl::getPersistdir() const
1409 {
1410 auto persdir(conf->getMainConfig().persistdir().getValue());
1411 if (persdir.back() != '/')
1412 persdir.push_back('/');
1413 std::string result = persdir + "repos/" + getHash();
1414 if (g_mkdir_with_parents(result.c_str(), 0755) == -1) {
1415 const char * errTxt = strerror(errno);
1416 throw RepoError(tfm::format(_("Cannot create persistdir \"%s\": %s"),
1417 result, errTxt));
1418 }
1419 return result;
1420 }
1421
getAge() const1422 int Repo::Impl::getAge() const
1423 {
1424 return time(NULL) - mtime(getMetadataPath(MD_TYPE_PRIMARY).c_str());
1425 }
1426
expire()1427 void Repo::Impl::expire()
1428 {
1429 expired = true;
1430 timestamp = 0;
1431 }
1432
isExpired() const1433 bool Repo::Impl::isExpired() const
1434 {
1435 if (expired)
1436 // explicitly requested expired state
1437 return true;
1438 if (conf->metadata_expire().getValue() == -1)
1439 return false;
1440 return getAge() > conf->metadata_expire().getValue();
1441 }
1442
getExpiresIn() const1443 int Repo::Impl::getExpiresIn() const
1444 {
1445 return conf->metadata_expire().getValue() - getAge();
1446 }
1447
downloadUrl(const char * url,int fd)1448 void Repo::Impl::downloadUrl(const char * url, int fd)
1449 {
1450 if (callbacks)
1451 callbacks->start(
1452 !conf->name().getValue().empty() ? conf->name().getValue().c_str() :
1453 (!id.empty() ? id.c_str() : "unknown")
1454 );
1455
1456 GError * errP{nullptr};
1457 lr_download_url(getCachedHandle(), url, fd, &errP);
1458 std::unique_ptr<GError> err(errP);
1459
1460 if (callbacks)
1461 callbacks->end();
1462
1463 if (err)
1464 throw LrExceptionWithSourceUrl(err->code, err->message, url);
1465 }
1466
setHttpHeaders(const char * headers[])1467 void Repo::Impl::setHttpHeaders(const char * headers[])
1468 {
1469 if (!headers) {
1470 httpHeaders.reset();
1471 return;
1472 }
1473 size_t headersCount = 0;
1474 while (headers[headersCount])
1475 ++headersCount;
1476 httpHeaders.reset(new char*[headersCount + 1]{});
1477 for (size_t i = 0; i < headersCount; ++i) {
1478 httpHeaders[i] = new char[strlen(headers[i]) + 1];
1479 strcpy(httpHeaders[i], headers[i]);
1480 }
1481 }
1482
getHttpHeaders() const1483 const char * const * Repo::Impl::getHttpHeaders() const
1484 {
1485 return httpHeaders.get();
1486 }
1487
fresh()1488 bool Repo::fresh()
1489 {
1490 return pImpl->fresh;
1491 }
1492
resetMetadataExpired()1493 void Repo::Impl::resetMetadataExpired()
1494 {
1495 if (expired || conf->metadata_expire().getValue() == -1)
1496 return;
1497 if (conf->getMainConfig().check_config_file_age().getValue() &&
1498 !repoFilePath.empty() &&
1499 mtime(repoFilePath.c_str()) > mtime(getMetadataPath(MD_TYPE_PRIMARY).c_str()))
1500 expired = true;
1501 else
1502 expired = getAge() > conf->metadata_expire().getValue();
1503 }
1504
1505
1506 /* Returns a librepo handle, set as per the repo options
1507 Note that destdir is None, and the handle is cached.*/
getCachedHandle()1508 LrHandle * Repo::Impl::getCachedHandle()
1509 {
1510 if (!handle)
1511 handle = lrHandleInitRemote(nullptr);
1512 handleSetOpt(handle.get(), LRO_HTTPHEADER, httpHeaders.get());
1513 return handle.get();
1514 }
1515
attachLibsolvRepo(LibsolvRepo * libsolvRepo)1516 void Repo::Impl::attachLibsolvRepo(LibsolvRepo * libsolvRepo)
1517 {
1518 std::lock_guard<std::mutex> guard(attachLibsolvMutex);
1519
1520 if (this->libsolvRepo)
1521 // A libsolvRepo was attached to this object before. Remove it's reference to this object.
1522 this->libsolvRepo->appdata = nullptr;
1523 else
1524 // The libsolvRepo will reference this object. Increase reference counter.
1525 ++nrefs;
1526
1527 libsolvRepo->appdata = owner; // The libsolvRepo references back to us.
1528 libsolvRepo->subpriority = -owner->getCost();
1529 libsolvRepo->priority = -owner->getPriority();
1530 this->libsolvRepo = libsolvRepo;
1531 }
1532
detachLibsolvRepo()1533 void Repo::Impl::detachLibsolvRepo()
1534 {
1535 attachLibsolvMutex.lock();
1536 if (!libsolvRepo) {
1537 // Nothing to do, libsolvRepo is not attached.
1538 attachLibsolvMutex.unlock();
1539 return;
1540 }
1541
1542 libsolvRepo->appdata = nullptr; // Removes reference to this object from libsolvRepo.
1543 this->libsolvRepo = nullptr;
1544
1545 if (--nrefs <= 0) {
1546 // There is no reference to this object, we are going to destroy it.
1547 // Mutex is part of this object, we must unlock it before destroying.
1548 attachLibsolvMutex.unlock();
1549 delete owner;
1550 } else
1551 attachLibsolvMutex.unlock();
1552 }
1553
stringToProxyAuthMethods(const std::string & proxyAuthMethodStr)1554 LrAuth Repo::Impl::stringToProxyAuthMethods(const std::string & proxyAuthMethodStr) noexcept
1555 {
1556 auto proxyAuthMethods = LR_AUTH_ANY;
1557 for (auto & auth : PROXYAUTHMETHODS) {
1558 if (proxyAuthMethodStr == auth.name) {
1559 proxyAuthMethods = auth.code;
1560 break;
1561 }
1562 }
1563 return proxyAuthMethods;
1564 }
1565
setMaxMirrorTries(int maxMirrorTries)1566 void Repo::setMaxMirrorTries(int maxMirrorTries)
1567 {
1568 pImpl->maxMirrorTries = maxMirrorTries;
1569 }
1570
getTimestamp() const1571 int Repo::getTimestamp() const
1572 {
1573 return pImpl->timestamp;
1574 }
1575
getMaxTimestamp()1576 int Repo::getMaxTimestamp()
1577 {
1578 return pImpl->maxTimestamp;
1579 }
1580
setPreserveRemoteTime(bool preserveRemoteTime)1581 void Repo::setPreserveRemoteTime(bool preserveRemoteTime)
1582 {
1583 pImpl->preserveRemoteTime = preserveRemoteTime;
1584 }
1585
getPreserveRemoteTime() const1586 bool Repo::getPreserveRemoteTime() const
1587 {
1588 return pImpl->preserveRemoteTime;
1589 }
1590
getContentTags()1591 const std::vector<std::string> & Repo::getContentTags()
1592 {
1593 return pImpl->content_tags;
1594 }
1595
getDistroTags()1596 const std::vector<std::pair<std::string, std::string>> & Repo::getDistroTags()
1597 {
1598 return pImpl->distro_tags;
1599 }
1600
getMetadataLocations() const1601 const std::vector<std::pair<std::string, std::string>> Repo::getMetadataLocations() const
1602 {
1603 return pImpl->metadata_locations;
1604 }
1605
getRevision() const1606 const std::string & Repo::getRevision() const
1607 {
1608 return pImpl->revision;
1609 }
1610
getCachedir() const1611 std::string Repo::getCachedir() const
1612 {
1613 return pImpl->getCachedir();
1614 }
1615
setRepoFilePath(const std::string & path)1616 void Repo::setRepoFilePath(const std::string & path)
1617 {
1618 pImpl->repoFilePath = path;
1619 }
1620
getRepoFilePath() const1621 const std::string & Repo::getRepoFilePath() const noexcept
1622 {
1623 return pImpl->repoFilePath;
1624 }
1625
setSyncStrategy(SyncStrategy strategy)1626 void Repo::setSyncStrategy(SyncStrategy strategy)
1627 {
1628 pImpl->syncStrategy = strategy;
1629 }
1630
getSyncStrategy() const1631 Repo::SyncStrategy Repo::getSyncStrategy() const noexcept
1632 {
1633 return pImpl->syncStrategy;
1634 }
1635
downloadUrl(const char * url,int fd)1636 void Repo::downloadUrl(const char * url, int fd)
1637 {
1638 pImpl->downloadUrl(url, fd);
1639 }
1640
setHttpHeaders(const char * headers[])1641 void Repo::setHttpHeaders(const char * headers[])
1642 {
1643 pImpl->setHttpHeaders(headers);
1644 }
1645
getHttpHeaders() const1646 const char * const * Repo::getHttpHeaders() const
1647 {
1648 return pImpl->getHttpHeaders();
1649 }
1650
getMirrors() const1651 std::vector<std::string> Repo::getMirrors() const
1652 {
1653 std::vector<std::string> mirrors;
1654 if (pImpl->mirrors) {
1655 for (auto mirror = pImpl->mirrors; *mirror; ++mirror)
1656 mirrors.emplace_back(*mirror);
1657 }
1658 return mirrors;
1659 }
1660
end(TransferStatus status,const char * msg)1661 int PackageTargetCB::end(TransferStatus status, const char * msg) { return 0; }
progress(double totalToDownload,double downloaded)1662 int PackageTargetCB::progress(double totalToDownload, double downloaded) { return 0; }
mirrorFailure(const char * msg,const char * url)1663 int PackageTargetCB::mirrorFailure(const char *msg, const char *url) { return 0; }
1664
1665 class PackageTarget::Impl {
1666 public:
1667 Impl(Repo * repo, const char * relativeUrl, const char * dest, int chksType,
1668 const char * chksum, int64_t expectedSize, const char * baseUrl, bool resume,
1669 int64_t byteRangeStart, int64_t byteRangeEnd, PackageTargetCB * callbacks);
1670
1671 Impl(ConfigMain * cfg, const char * relativeUrl, const char * dest, int chksType,
1672 const char * chksum, int64_t expectedSize, const char * baseUrl, bool resume,
1673 int64_t byteRangeStart, int64_t byteRangeEnd, PackageTargetCB * callbacks,
1674 const char * httpHeaders[]);
1675
1676 void download();
1677
1678 ~Impl();
1679
1680 PackageTargetCB * callbacks;
1681
1682 std::unique_ptr<LrPackageTarget> lrPkgTarget;
1683
1684 private:
1685 void init(LrHandle * handle, const char * relativeUrl, const char * dest, int chksType,
1686 const char * chksum, int64_t expectedSize, const char * baseUrl, bool resume,
1687 int64_t byteRangeStart, int64_t byteRangeEnd);
1688
1689 static int endCB(void * data, LrTransferStatus status, const char * msg);
1690 static int progressCB(void * data, double totalToDownload, double downloaded);
1691 static int mirrorFailureCB(void * data, const char * msg, const char * url);
1692
1693 std::unique_ptr<LrHandle> lrHandle;
1694
1695 };
1696
1697
endCB(void * data,LrTransferStatus status,const char * msg)1698 int PackageTarget::Impl::endCB(void * data, LrTransferStatus status, const char * msg)
1699 {
1700 if (!data)
1701 return 0;
1702 auto cbObject = static_cast<PackageTargetCB *>(data);
1703 return cbObject->end(static_cast<PackageTargetCB::TransferStatus>(status), msg);
1704 }
1705
progressCB(void * data,double totalToDownload,double downloaded)1706 int PackageTarget::Impl::progressCB(void * data, double totalToDownload, double downloaded)
1707 {
1708 if (!data)
1709 return 0;
1710 auto cbObject = static_cast<PackageTargetCB *>(data);
1711 return cbObject->progress(totalToDownload, downloaded);
1712 }
1713
mirrorFailureCB(void * data,const char * msg,const char * url)1714 int PackageTarget::Impl::mirrorFailureCB(void * data, const char * msg, const char * url)
1715 {
1716 if (!data)
1717 return 0;
1718 auto cbObject = static_cast<PackageTargetCB *>(data);
1719 return cbObject->mirrorFailure(msg, url);
1720 }
1721
1722
newHandle(ConfigMain * conf)1723 static LrHandle * newHandle(ConfigMain * conf)
1724 {
1725 LrHandle *h = lr_handle_init();
1726 const char * user_agent = USER_AGENT;
1727 // see dnf.repo.Repo._handle_new_remote() how to pass
1728 if (conf) {
1729 user_agent = conf->user_agent().getValue().c_str();
1730 setHandle(h, *conf);
1731 }
1732 handleSetOpt(h, LRO_USERAGENT, user_agent);
1733 return h;
1734 }
1735
checksumType(const std::string & name)1736 PackageTarget::ChecksumType PackageTarget::checksumType(const std::string & name)
1737 {
1738 return static_cast<ChecksumType>(lr_checksum_type(name.c_str()));
1739 }
1740
downloadPackages(std::vector<PackageTarget * > & targets,bool failFast)1741 void PackageTarget::downloadPackages(std::vector<PackageTarget *> & targets, bool failFast)
1742 {
1743 // Convert vector to GSList
1744 GSList * list{nullptr};
1745 for (auto it = targets.rbegin(); it != targets.rend(); ++it)
1746 list = g_slist_prepend(list, (*it)->pImpl->lrPkgTarget.get());
1747 std::unique_ptr<GSList, decltype(&g_slist_free)> listGuard(list, &g_slist_free);
1748
1749 LrPackageDownloadFlag flags = static_cast<LrPackageDownloadFlag>(0);
1750 if (failFast)
1751 flags = static_cast<LrPackageDownloadFlag>(flags | LR_PACKAGEDOWNLOAD_FAILFAST);
1752
1753 GError * errP{nullptr};
1754 lr_download_packages(list, flags, &errP);
1755 std::unique_ptr<GError> err(errP);
1756
1757 if (err)
1758 throwException(std::move(err));
1759 }
1760
1761
~Impl()1762 PackageTarget::Impl::~Impl() {}
1763
Impl(Repo * repo,const char * relativeUrl,const char * dest,int chksType,const char * chksum,int64_t expectedSize,const char * baseUrl,bool resume,int64_t byteRangeStart,int64_t byteRangeEnd,PackageTargetCB * callbacks)1764 PackageTarget::Impl::Impl(Repo * repo, const char * relativeUrl, const char * dest, int chksType,
1765 const char * chksum, int64_t expectedSize, const char * baseUrl, bool resume,
1766 int64_t byteRangeStart, int64_t byteRangeEnd, PackageTargetCB * callbacks)
1767 : callbacks(callbacks)
1768 {
1769 init(repo->pImpl->getCachedHandle(), relativeUrl, dest, chksType, chksum, expectedSize,
1770 baseUrl, resume, byteRangeStart, byteRangeEnd);
1771 }
1772
Impl(ConfigMain * cfg,const char * relativeUrl,const char * dest,int chksType,const char * chksum,int64_t expectedSize,const char * baseUrl,bool resume,int64_t byteRangeStart,int64_t byteRangeEnd,PackageTargetCB * callbacks,const char * httpHeaders[])1773 PackageTarget::Impl::Impl(ConfigMain * cfg, const char * relativeUrl, const char * dest, int chksType,
1774 const char * chksum, int64_t expectedSize, const char * baseUrl, bool resume,
1775 int64_t byteRangeStart, int64_t byteRangeEnd, PackageTargetCB * callbacks,
1776 const char * httpHeaders[])
1777 : callbacks(callbacks)
1778 {
1779 lrHandle.reset(newHandle(cfg));
1780 handleSetOpt(lrHandle.get(), LRO_HTTPHEADER, httpHeaders);
1781 handleSetOpt(lrHandle.get(), LRO_REPOTYPE, LR_YUMREPO);
1782 init(lrHandle.get(), relativeUrl, dest, chksType, chksum, expectedSize, baseUrl, resume,
1783 byteRangeStart, byteRangeEnd);
1784 }
1785
init(LrHandle * handle,const char * relativeUrl,const char * dest,int chksType,const char * chksum,int64_t expectedSize,const char * baseUrl,bool resume,int64_t byteRangeStart,int64_t byteRangeEnd)1786 void PackageTarget::Impl::init(LrHandle * handle, const char * relativeUrl, const char * dest, int chksType,
1787 const char * chksum, int64_t expectedSize, const char * baseUrl, bool resume,
1788 int64_t byteRangeStart, int64_t byteRangeEnd)
1789 {
1790 LrChecksumType lrChksType = static_cast<LrChecksumType>(chksType);
1791
1792 if (resume && byteRangeStart) {
1793 auto msg = _("resume cannot be used simultaneously with the byterangestart param");
1794 throw Exception(msg);
1795 }
1796
1797 GError * errP{nullptr};
1798
1799 std::string encodedUrl = relativeUrl;
1800 if (encodedUrl.find("://") == std::string::npos) {
1801 encodedUrl = urlEncode(encodedUrl, "/");
1802 }
1803
1804 lrPkgTarget.reset(lr_packagetarget_new_v3(handle, encodedUrl.c_str(), dest, lrChksType, chksum,
1805 expectedSize, baseUrl, resume, progressCB, callbacks, endCB,
1806 mirrorFailureCB, byteRangeStart, byteRangeEnd, &errP));
1807 std::unique_ptr<GError> err(errP);
1808
1809 if (!lrPkgTarget) {
1810 auto msg = tfm::format(_("PackageTarget initialization failed: %s"), err->message);
1811 throw Exception(msg);
1812 }
1813 }
1814
PackageTarget(Repo * repo,const char * relativeUrl,const char * dest,int chksType,const char * chksum,int64_t expectedSize,const char * baseUrl,bool resume,int64_t byteRangeStart,int64_t byteRangeEnd,PackageTargetCB * callbacks)1815 PackageTarget::PackageTarget(Repo * repo, const char * relativeUrl, const char * dest, int chksType,
1816 const char * chksum, int64_t expectedSize, const char * baseUrl, bool resume,
1817 int64_t byteRangeStart, int64_t byteRangeEnd, PackageTargetCB * callbacks)
1818 : pImpl(new Impl(repo, relativeUrl, dest, chksType, chksum, expectedSize, baseUrl, resume,
1819 byteRangeStart, byteRangeEnd, callbacks))
1820 {}
1821
PackageTarget(ConfigMain * cfg,const char * relativeUrl,const char * dest,int chksType,const char * chksum,int64_t expectedSize,const char * baseUrl,bool resume,int64_t byteRangeStart,int64_t byteRangeEnd,PackageTargetCB * callbacks,const char * httpHeaders[])1822 PackageTarget::PackageTarget(ConfigMain * cfg, const char * relativeUrl, const char * dest, int chksType,
1823 const char * chksum, int64_t expectedSize, const char * baseUrl, bool resume,
1824 int64_t byteRangeStart, int64_t byteRangeEnd, PackageTargetCB * callbacks,
1825 const char * httpHeaders[])
1826 : pImpl(new Impl(cfg, relativeUrl, dest, chksType, chksum, expectedSize, baseUrl, resume,
1827 byteRangeStart, byteRangeEnd, callbacks, httpHeaders))
1828 {}
1829
1830
~PackageTarget()1831 PackageTarget::~PackageTarget() {}
1832
getCallbacks()1833 PackageTargetCB * PackageTarget::getCallbacks()
1834 {
1835 return pImpl->callbacks;
1836 }
1837
getErr()1838 const char * PackageTarget::getErr()
1839 {
1840 return pImpl->lrPkgTarget->err;
1841 }
1842
downloadURL(ConfigMain * cfg,const char * url,int fd)1843 void Downloader::downloadURL(ConfigMain * cfg, const char * url, int fd)
1844 {
1845 std::unique_ptr<LrHandle> lrHandle(newHandle(cfg));
1846 GError * errP{nullptr};
1847 lr_download_url(lrHandle.get(), url, fd, &errP);
1848 std::unique_ptr<GError> err(errP);
1849
1850 if (err)
1851 throwException(std::move(err));
1852 }
1853
1854 // ============ librepo logging ===========
1855
1856 #define LR_LOGDOMAIN "librepo"
1857
1858 class LrHandleLogData {
1859 public:
1860 std::string filePath;
1861 long uid;
1862 FILE *fd;
1863 bool used{false};
1864 guint handlerId;
1865
1866 ~LrHandleLogData();
1867 };
1868
~LrHandleLogData()1869 LrHandleLogData::~LrHandleLogData()
1870 {
1871 if (used)
1872 g_log_remove_handler(LR_LOGDOMAIN, handlerId);
1873 fclose(fd);
1874 }
1875
1876 static std::list<std::unique_ptr<LrHandleLogData>> lrLogDatas;
1877 static std::mutex lrLogDatasMutex;
1878
lrLogLevelFlagToLevel(GLogLevelFlags logLevelFlag)1879 static const Logger::Level lrLogLevelFlagToLevel(GLogLevelFlags logLevelFlag)
1880 {
1881 if (logLevelFlag & G_LOG_LEVEL_ERROR) {
1882 return Logger::Level::ERROR;
1883 }
1884 if (logLevelFlag & G_LOG_LEVEL_CRITICAL) {
1885 return Logger::Level::WARNING;
1886 }
1887 if (logLevelFlag & G_LOG_LEVEL_WARNING) {
1888 return Logger::Level::WARNING;
1889 }
1890 if (logLevelFlag & G_LOG_LEVEL_MESSAGE) {
1891 return Logger::Level::NOTICE;
1892 }
1893 if (logLevelFlag & G_LOG_LEVEL_INFO) {
1894 return Logger::Level::INFO;
1895 }
1896 if (logLevelFlag & G_LOG_LEVEL_DEBUG) {
1897 return Logger::Level::DEBUG;
1898 }
1899 return Logger::Level::TRACE;
1900 }
1901
librepoLogCB(G_GNUC_UNUSED const gchar * log_domain,GLogLevelFlags log_level,const char * msg,gpointer user_data)1902 static void librepoLogCB(G_GNUC_UNUSED const gchar *log_domain, GLogLevelFlags log_level,
1903 const char *msg, gpointer user_data) noexcept
1904 {
1905 auto logger(Log::getLogger());
1906 logger->write(Logger::LOG_SOURCE_LIBREPO, lrLogLevelFlagToLevel(log_level), msg);
1907 }
1908
addHandler(const std::string & filePath,bool debug)1909 long LibrepoLog::addHandler(const std::string & filePath, bool debug)
1910 {
1911 static long uid = 0;
1912
1913 // Open the file
1914 FILE *fd = fopen(filePath.c_str(), "a");
1915 if (!fd)
1916 throw RepoError(tfm::format(_("Cannot open %s: %s"), filePath, g_strerror(errno)));
1917
1918 // Setup user data
1919 std::unique_ptr<LrHandleLogData> data(new LrHandleLogData);
1920 data->filePath = filePath;
1921 data->fd = fd;
1922
1923 // Set handler
1924 GLogLevelFlags log_mask = debug ? G_LOG_LEVEL_MASK : static_cast<GLogLevelFlags>(
1925 G_LOG_LEVEL_INFO |
1926 G_LOG_LEVEL_MESSAGE |
1927 G_LOG_LEVEL_WARNING |
1928 G_LOG_LEVEL_CRITICAL |
1929 G_LOG_LEVEL_ERROR);
1930
1931 data->handlerId = g_log_set_handler(LR_LOGDOMAIN, log_mask, librepoLogCB, data.get());
1932 data->used = true;
1933
1934 // Save user data (in a thread safe way)
1935 {
1936 std::lock_guard<std::mutex> guard(lrLogDatasMutex);
1937
1938 // Get unique ID of the handler
1939 data->uid = ++uid;
1940
1941 // Append the data to the global list
1942 lrLogDatas.push_front(std::move(data));
1943 }
1944
1945 // Log librepo version and current time (including timezone)
1946 lr_log_librepo_summary();
1947
1948 // Return unique id of the handler data
1949 return uid;
1950 }
1951
removeHandler(long uid)1952 void LibrepoLog::removeHandler(long uid)
1953 {
1954 std::lock_guard<std::mutex> guard(lrLogDatasMutex);
1955
1956 // Search for the corresponding LogFileData
1957 auto it = lrLogDatas.begin();
1958 for (; it != lrLogDatas.end() && (*it)->uid != uid; ++it);
1959 if (it == lrLogDatas.end())
1960 throw Exception(tfm::format(_("Log handler with id %ld doesn't exist"), uid));
1961
1962 // Remove the handler and free the data
1963 lrLogDatas.erase(it);
1964 }
1965
removeAllHandlers()1966 void LibrepoLog::removeAllHandlers()
1967 {
1968 std::lock_guard<std::mutex> guard(lrLogDatasMutex);
1969 lrLogDatas.clear();
1970 }
1971
repoGetImpl(Repo * repo)1972 Repo::Impl * repoGetImpl(Repo * repo)
1973 {
1974 return repo->pImpl.get();
1975 }
1976
1977 }
1978
1979 // hawkey
1980 #include "../hy-repo-private.hpp"
1981
1982 void
repo_internalize_all_trigger(Pool * pool)1983 repo_internalize_all_trigger(Pool *pool)
1984 {
1985 int i;
1986 Repo *repo;
1987
1988 FOR_REPOS(i, repo)
1989 repo_internalize_trigger(repo);
1990 }
1991
1992 void
repo_internalize_trigger(Repo * repo)1993 repo_internalize_trigger(Repo * repo)
1994 {
1995 if (!repo)
1996 return;
1997
1998 if (auto hrepo = static_cast<HyRepo>(repo->appdata)) {
1999 // HyRepo is attached. The hint needs_internalizing will be used.
2000 auto repoImpl = libdnf::repoGetImpl(hrepo);
2001 assert(repoImpl->libsolvRepo == repo);
2002 if (!repoImpl->needs_internalizing)
2003 return;
2004 repoImpl->needs_internalizing = false;
2005 }
2006
2007 repo_internalize(repo);
2008 }
2009
2010 void
repo_update_state(HyRepo repo,enum _hy_repo_repodata which,enum _hy_repo_state state)2011 repo_update_state(HyRepo repo, enum _hy_repo_repodata which,
2012 enum _hy_repo_state state)
2013 {
2014 auto repoImpl = libdnf::repoGetImpl(repo);
2015 assert(state <= _HY_WRITTEN);
2016 switch (which) {
2017 case _HY_REPODATA_FILENAMES:
2018 repoImpl->state_filelists = state;
2019 return;
2020 case _HY_REPODATA_PRESTO:
2021 repoImpl->state_presto = state;
2022 return;
2023 case _HY_REPODATA_UPDATEINFO:
2024 repoImpl->state_updateinfo = state;
2025 return;
2026 case _HY_REPODATA_OTHER:
2027 repoImpl->state_other = state;
2028 return;
2029 default:
2030 assert(0);
2031 }
2032 return;
2033 }
2034
2035 Id
repo_get_repodata(HyRepo repo,enum _hy_repo_repodata which)2036 repo_get_repodata(HyRepo repo, enum _hy_repo_repodata which)
2037 {
2038 auto repoImpl = libdnf::repoGetImpl(repo);
2039 switch (which) {
2040 case _HY_REPODATA_FILENAMES:
2041 return repoImpl->filenames_repodata;
2042 case _HY_REPODATA_PRESTO:
2043 return repoImpl->presto_repodata;
2044 case _HY_REPODATA_UPDATEINFO:
2045 return repoImpl->updateinfo_repodata;
2046 case _HY_REPODATA_OTHER:
2047 return repoImpl->other_repodata;
2048 default:
2049 assert(0);
2050 return 0;
2051 }
2052 }
2053
2054 void
repo_set_repodata(HyRepo repo,enum _hy_repo_repodata which,Id repodata)2055 repo_set_repodata(HyRepo repo, enum _hy_repo_repodata which, Id repodata)
2056 {
2057 auto repoImpl = libdnf::repoGetImpl(repo);
2058 switch (which) {
2059 case _HY_REPODATA_FILENAMES:
2060 repoImpl->filenames_repodata = repodata;
2061 return;
2062 case _HY_REPODATA_PRESTO:
2063 repoImpl->presto_repodata = repodata;
2064 return;
2065 case _HY_REPODATA_UPDATEINFO:
2066 repoImpl->updateinfo_repodata = repodata;
2067 return;
2068 case _HY_REPODATA_OTHER:
2069 repoImpl->other_repodata = repodata;
2070 return;
2071 default:
2072 assert(0);
2073 return;
2074 }
2075 }
2076
2077 // public functions
2078
2079 HyRepo
hy_repo_create(const char * name)2080 hy_repo_create(const char *name)
2081 {
2082 assert(name);
2083 auto & cfgMain = libdnf::getGlobalMainConfig();
2084 std::unique_ptr<libdnf::ConfigRepo> cfgRepo(new libdnf::ConfigRepo(cfgMain));
2085 auto repo = new libdnf::Repo(name, std::move(cfgRepo), libdnf::Repo::Type::COMMANDLINE);
2086 auto repoImpl = libdnf::repoGetImpl(repo);
2087 repoImpl->conf->name().set(libdnf::Option::Priority::RUNTIME, name);
2088 return repo;
2089 }
2090
2091 int
hy_repo_get_cost(HyRepo repo)2092 hy_repo_get_cost(HyRepo repo)
2093 {
2094 return repo->getCost();
2095 }
2096
2097 int
hy_repo_get_priority(HyRepo repo)2098 hy_repo_get_priority(HyRepo repo)
2099 {
2100 return repo->getPriority();
2101 }
2102
2103 gboolean
hy_repo_get_use_includes(HyRepo repo)2104 hy_repo_get_use_includes(HyRepo repo)
2105 {
2106 return repo->getUseIncludes();
2107 }
2108
2109 guint
hy_repo_get_n_solvables(HyRepo repo)2110 hy_repo_get_n_solvables(HyRepo repo)
2111 {
2112 return (guint)libdnf::repoGetImpl(repo)->libsolvRepo->nsolvables;
2113 }
2114
2115 void
hy_repo_set_cost(HyRepo repo,int value)2116 hy_repo_set_cost(HyRepo repo, int value)
2117 {
2118 auto repoImpl = libdnf::repoGetImpl(repo);
2119 repoImpl->conf->cost().set(libdnf::Option::Priority::RUNTIME, value);
2120 if (repoImpl->libsolvRepo)
2121 repoImpl->libsolvRepo->subpriority = -value;
2122 }
2123
2124 void
hy_repo_set_priority(HyRepo repo,int value)2125 hy_repo_set_priority(HyRepo repo, int value)
2126 {
2127 auto repoImpl = libdnf::repoGetImpl(repo);
2128 repoImpl->conf->priority().set(libdnf::Option::Priority::RUNTIME, value);
2129 if (repoImpl->libsolvRepo)
2130 repoImpl->libsolvRepo->priority = -value;
2131 }
2132
2133 void
hy_repo_set_use_includes(HyRepo repo,gboolean enabled)2134 hy_repo_set_use_includes(HyRepo repo, gboolean enabled)
2135 {
2136 repo->setUseIncludes(enabled);
2137 }
2138
2139 void
hy_repo_set_string(HyRepo repo,int which,const char * str_val)2140 hy_repo_set_string(HyRepo repo, int which, const char *str_val)
2141 {
2142 auto repoImpl = libdnf::repoGetImpl(repo);
2143 switch (which) {
2144 case HY_REPO_NAME:
2145 repoImpl->id = str_val;
2146 repoImpl->conf->name().set(libdnf::Option::Priority::RUNTIME, str_val);
2147 break;
2148 case HY_REPO_MD_FN:
2149 repoImpl->repomdFn = str_val ? str_val : "";
2150 break;
2151 case HY_REPO_PRIMARY_FN:
2152 repoImpl->metadataPaths[MD_TYPE_PRIMARY] = str_val ? str_val : "";
2153 break;
2154 case HY_REPO_FILELISTS_FN:
2155 repoImpl->metadataPaths[MD_TYPE_FILELISTS] = str_val ? str_val : "";
2156 break;
2157 case HY_REPO_PRESTO_FN:
2158 repoImpl->metadataPaths[MD_TYPE_PRESTODELTA] = str_val ? str_val : "";
2159 break;
2160 case HY_REPO_UPDATEINFO_FN:
2161 repoImpl->metadataPaths[MD_TYPE_UPDATEINFO] = str_val ? str_val : "";
2162 break;
2163 case HY_REPO_OTHER_FN:
2164 repoImpl->metadataPaths[MD_TYPE_OTHER] = str_val ? str_val : "";
2165 break;
2166 case MODULES_FN:
2167 repoImpl->metadataPaths[MD_TYPE_MODULES] = str_val ? str_val : "";
2168 break;
2169 default:
2170 assert(0);
2171 }
2172 }
2173
2174 const char *
hy_repo_get_string(HyRepo repo,int which)2175 hy_repo_get_string(HyRepo repo, int which)
2176 {
2177 auto repoImpl = libdnf::repoGetImpl(repo);
2178 const char * ret;
2179 switch(which) {
2180 case HY_REPO_NAME:
2181 return repoImpl->id.c_str();
2182 case HY_REPO_MD_FN:
2183 ret = repoImpl->repomdFn.c_str();
2184 break;
2185 case HY_REPO_PRIMARY_FN:
2186 ret = repoImpl->getMetadataPath(MD_TYPE_PRIMARY).c_str();
2187 break;
2188 case HY_REPO_FILELISTS_FN:
2189 ret = repoImpl->getMetadataPath(MD_TYPE_FILELISTS).c_str();
2190 break;
2191 case HY_REPO_PRESTO_FN:
2192 ret = repoImpl->getMetadataPath(MD_TYPE_PRESTODELTA).c_str();
2193 break;
2194 case HY_REPO_UPDATEINFO_FN:
2195 ret = repoImpl->getMetadataPath(MD_TYPE_UPDATEINFO).c_str();
2196 break;
2197 case HY_REPO_OTHER_FN:
2198 ret = repoImpl->getMetadataPath(MD_TYPE_OTHER).c_str();
2199 break;
2200 case MODULES_FN:
2201 ret = repoImpl->getMetadataPath(MD_TYPE_MODULES).c_str();
2202 break;
2203 default:
2204 return nullptr;
2205 }
2206 return ret[0] == '\0' ? nullptr : ret;
2207 }
2208
2209 void
hy_repo_free(HyRepo repo)2210 hy_repo_free(HyRepo repo)
2211 {
2212 auto repoImpl = libdnf::repoGetImpl(repo);
2213 {
2214 std::lock_guard<std::mutex> guard(repoImpl->attachLibsolvMutex);
2215 if (--repoImpl->nrefs > 0)
2216 return; // There is still a reference to this object. Don't destroy it.
2217 }
2218 assert(!repoImpl->libsolvRepo);
2219 delete repo;
2220 }
2221