libzypp 17.31.20
MediaCurl.cc
Go to the documentation of this file.
1/*---------------------------------------------------------------------\
2| ____ _ __ __ ___ |
3| |__ / \ / / . \ . \ |
4| / / \ V /| _/ _/ |
5| / /__ | | | | | | |
6| /_____||_| |_| |_| |
7| |
8\---------------------------------------------------------------------*/
13#include <iostream>
14#include <chrono>
15#include <list>
16
17#include <zypp/base/Logger.h>
18#include <zypp/ExternalProgram.h>
19#include <zypp/base/String.h>
20#include <zypp/base/Gettext.h>
21#include <zypp-core/parser/Sysconfig>
22#include <zypp/base/Gettext.h>
23
25#include <zypp-curl/ProxyInfo>
26#include <zypp-curl/auth/CurlAuthData>
27#include <zypp-media/auth/CredentialManager>
28#include <zypp-curl/CurlConfig>
30#include <zypp/Target.h>
31#include <zypp/ZYppFactory.h>
32#include <zypp/ZConfig.h>
33
34#include <cstdlib>
35#include <sys/types.h>
36#include <sys/stat.h>
37#include <sys/mount.h>
38#include <errno.h>
39#include <dirent.h>
40#include <unistd.h>
41
42using std::endl;
43
44namespace internal {
45 using namespace zypp;
49 struct OptionalDownloadProgressReport : public callback::ReceiveReport<media::DownloadProgressReport>
50 {
51 using TimePoint = std::chrono::steady_clock::time_point;
52
53 OptionalDownloadProgressReport( bool isOptional=false )
54 : _oldRec { Distributor::instance().getReceiver() }
55 , _isOptional { isOptional }
56 { connect(); }
57
60
61 void reportbegin() override
62 { if ( _oldRec ) _oldRec->reportbegin(); }
63
64 void reportend() override
65 { if ( _oldRec ) _oldRec->reportend(); }
66
67 void report( const UserData & userData_r = UserData() ) override
68 { if ( _oldRec ) _oldRec->report( userData_r ); }
69
70
71 void start( const Url & file_r, Pathname localfile_r ) override
72 {
73 if ( not _oldRec ) return;
74 if ( _isOptional ) {
75 // delay start until first data are received.
76 _startFile = file_r;
77 _startLocalfile = std::move(localfile_r);
78 return;
79 }
80 _oldRec->start( file_r, localfile_r );
81 }
82
83 bool progress( int value_r, const Url & file_r, double dbps_avg_r = -1, double dbps_current_r = -1 ) override
84 {
85 if ( not _oldRec ) return true;
86 if ( notStarted() ) {
87 if ( not ( value_r || dbps_avg_r || dbps_current_r ) )
88 return true;
89 sendStart();
90 }
91
92 //static constexpr std::chrono::milliseconds minfequency { 1000 }; only needed if we'd avoid sending reports without change
93 static constexpr std::chrono::milliseconds maxfequency { 100 };
94 TimePoint now { TimePoint::clock::now() };
95 TimePoint::duration elapsed { now - _lastProgressSent };
96 if ( elapsed < maxfequency )
97 return true; // continue
99 return _oldRec->progress( value_r, file_r, dbps_avg_r, dbps_current_r );
100 }
101
102 Action problem( const Url & file_r, Error error_r, const std::string & description_r ) override
103 {
104 if ( not _oldRec || notStarted() ) return ABORT;
105 return _oldRec->problem( file_r, error_r, description_r );
106 }
107
108 void finish( const Url & file_r, Error error_r, const std::string & reason_r ) override
109 {
110 if ( not _oldRec || notStarted() ) return;
111 _oldRec->finish( file_r, error_r, reason_r );
112 }
113
114 private:
115 // _isOptional also indicates the delayed start
116 bool notStarted() const
117 { return _isOptional; }
118
120 {
121 if ( _isOptional ) {
122 // we know _oldRec is valid...
123 _oldRec->start( std::move(_startFile), std::move(_startLocalfile) );
124 _isOptional = false;
125 }
126 }
127
128 private:
134 };
135
137 {
138 ProgressData( CURL *curl, time_t timeout = 0, const zypp::Url & url = zypp::Url(),
139 zypp::ByteCount expectedFileSize_r = 0,
141
142 void updateStats( double dltotal = 0.0, double dlnow = 0.0 );
143
144 int reportProgress() const;
145
146 CURL * curl()
147 { return _curl; }
148
149 bool timeoutReached() const
150 { return _timeoutReached; }
151
152 bool fileSizeExceeded() const
153 { return _fileSizeExceeded; }
154
156 { return _expectedFileSize; }
157
159 { _expectedFileSize = newval_r; }
160
161 private:
162 CURL * _curl;
164 time_t _timeout;
169
170 time_t _timeStart = 0;
171 time_t _timeLast = 0;
172 time_t _timeRcv = 0;
173 time_t _timeNow = 0;
174
175 double _dnlTotal = 0.0;
176 double _dnlLast = 0.0;
177 double _dnlNow = 0.0;
178
179 int _dnlPercent= 0;
180
181 double _drateTotal= 0.0;
182 double _drateLast = 0.0;
183 };
184
185
186
187 ProgressData::ProgressData(CURL *curl, time_t timeout, const Url &url, ByteCount expectedFileSize_r, zypp::callback::SendReport< zypp::media::DownloadProgressReport> *_report)
188 : _curl( curl )
189 , _url( url )
190 , _timeout( timeout )
191 , _timeoutReached( false )
192 , _fileSizeExceeded ( false )
193 , _expectedFileSize( expectedFileSize_r )
194 , report( _report )
195 {}
196
197 void ProgressData::updateStats(double dltotal, double dlnow)
198 {
199 time_t now = _timeNow = time(0);
200
201 // If called without args (0.0), recompute based on the last values seen
202 if ( dltotal && dltotal != _dnlTotal )
203 _dnlTotal = dltotal;
204
205 if ( dlnow && dlnow != _dnlNow )
206 {
207 _timeRcv = now;
208 _dnlNow = dlnow;
209 }
210
211 // init or reset if time jumps back
212 if ( !_timeStart || _timeStart > now )
213 _timeStart = _timeLast = _timeRcv = now;
214
215 // timeout condition
216 if ( _timeout )
217 _timeoutReached = ( (now - _timeRcv) > _timeout );
218
219 // check if the downloaded data is already bigger than what we expected
220 _fileSizeExceeded = _expectedFileSize > 0 && _expectedFileSize < static_cast<ByteCount::SizeType>(_dnlNow);
221
222 // percentage:
223 if ( _dnlTotal )
224 _dnlPercent = int(_dnlNow * 100 / _dnlTotal);
225
226 // download rates:
227 _drateTotal = _dnlNow / std::max( int(now - _timeStart), 1 );
228
229 if ( _timeLast < now )
230 {
231 _drateLast = (_dnlNow - _dnlLast) / int(now - _timeLast);
232 // start new period
233 _timeLast = now;
235 }
236 else if ( _timeStart == _timeLast )
238 }
239
241 {
242 if ( _fileSizeExceeded )
243 return 1;
244 if ( _timeoutReached )
245 return 1; // no-data timeout
246 if ( report && !(*report)->progress( _dnlPercent, _url, _drateTotal, _drateLast ) )
247 return 1; // user requested abort
248 return 0;
249 }
250
251 const char * anonymousIdHeader()
252 {
253 // we need to add the release and identifier to the
254 // agent string.
255 // The target could be not initialized, and then this information
256 // is guessed.
257 // bsc#1212187: HTTP/2 RFC 9113 forbids fields ending with a space
258 static const std::string _value( str::trim( str::form(
259 "X-ZYpp-AnonymousId: %s",
260 Target::anonymousUniqueId( Pathname()/*guess root*/ ).c_str()
261 )));
262 return _value.c_str();
263 }
264
266 {
267 // we need to add the release and identifier to the
268 // agent string.
269 // The target could be not initialized, and then this information
270 // is guessed.
271 // bsc#1212187: HTTP/2 RFC 9113 forbids fields ending with a space
272 static const std::string _value( str::trim( str::form(
273 "X-ZYpp-DistributionFlavor: %s",
274 Target::distributionFlavor( Pathname()/*guess root*/ ).c_str()
275 )));
276 return _value.c_str();
277 }
278
279 const char * agentString()
280 {
281 // we need to add the release and identifier to the
282 // agent string.
283 // The target could be not initialized, and then this information
284 // is guessed.
285 // bsc#1212187: HTTP/2 RFC 9113 forbids fields ending with a space
286 static const std::string _value( str::trim( str::form(
287 "ZYpp " LIBZYPP_VERSION_STRING " (curl %s) %s"
288 , curl_version_info(CURLVERSION_NOW)->version
289 , Target::targetDistribution( Pathname()/*guess root*/ ).c_str()
290 )));
291 return _value.c_str();
292 }
293
298 {
299 public:
301 const std::string & err_r,
302 const std::string & msg_r )
303 : media::MediaCurlException( url_r, err_r, msg_r )
304 {}
305 //~MediaCurlExceptionMayRetryInternaly() noexcept {}
306 };
307
308}
309
310
311using namespace internal;
312using namespace zypp::base;
313
314namespace zypp {
315
316 namespace media {
317
318Pathname MediaCurl::_cookieFile = "/var/lib/YaST2/cookies";
319
320// we use this define to unbloat code as this C setting option
321// and catching exception is done frequently.
323#define SET_OPTION(opt,val) do { \
324 ret = curl_easy_setopt ( _curl, opt, val ); \
325 if ( ret != 0) { \
326 ZYPP_THROW(MediaCurlSetOptException(_url, _curlError)); \
327 } \
328 } while ( false )
329
330#define SET_OPTION_OFFT(opt,val) SET_OPTION(opt,(curl_off_t)val)
331#define SET_OPTION_LONG(opt,val) SET_OPTION(opt,(long)val)
332#define SET_OPTION_VOID(opt,val) SET_OPTION(opt,(void*)val)
333
335 const Pathname & attach_point_hint_r )
336 : MediaNetworkCommonHandler( url_r, attach_point_hint_r,
337 "/", // urlpath at attachpoint
338 true ), // does_download
339 _curl( NULL ),
340 _customHeaders(0L)
341{
342 _curlError[0] = '\0';
343
344 MIL << "MediaCurl::MediaCurl(" << url_r << ", " << attach_point_hint_r << ")" << endl;
345
347
348 if( !attachPoint().empty())
349 {
350 PathInfo ainfo(attachPoint());
351 Pathname apath(attachPoint() + "XXXXXX");
352 char *atemp = ::strdup( apath.asString().c_str());
353 char *atest = NULL;
354 if( !ainfo.isDir() || !ainfo.userMayRWX() ||
355 atemp == NULL || (atest=::mkdtemp(atemp)) == NULL)
356 {
357 WAR << "attach point " << ainfo.path()
358 << " is not useable for " << url_r.getScheme() << endl;
359 setAttachPoint("", true);
360 }
361 else if( atest != NULL)
362 ::rmdir(atest);
363
364 if( atemp != NULL)
365 ::free(atemp);
366 }
367}
368
370{
372}
373
374void MediaCurl::setCookieFile( const Pathname &fileName )
375{
376 _cookieFile = fileName;
377}
378
380
381void MediaCurl::checkProtocol(const Url &url) const
382{
383 curl_version_info_data *curl_info = NULL;
384 curl_info = curl_version_info(CURLVERSION_NOW);
385 // curl_info does not need any free (is static)
386 if (curl_info->protocols)
387 {
388 const char * const *proto;
389 std::string scheme( url.getScheme());
390 bool found = false;
391 for(proto=curl_info->protocols; !found && *proto; ++proto)
392 {
393 if( scheme == std::string((const char *)*proto))
394 found = true;
395 }
396 if( !found)
397 {
398 std::string msg("Unsupported protocol '");
399 msg += scheme;
400 msg += "'";
402 }
403 }
404}
405
407{
409
410 curl_easy_setopt(_curl, CURLOPT_HEADERFUNCTION, log_redirects_curl);
411 curl_easy_setopt(_curl, CURLOPT_HEADERDATA, &_lastRedirect);
412 CURLcode ret = curl_easy_setopt( _curl, CURLOPT_ERRORBUFFER, _curlError );
413 if ( ret != 0 ) {
414 ZYPP_THROW(MediaCurlSetOptException(_url, "Error setting error buffer"));
415 }
416
417 SET_OPTION(CURLOPT_FAILONERROR, 1L);
418 SET_OPTION(CURLOPT_NOSIGNAL, 1L);
419
420 // create non persistant settings
421 // so that we don't add headers twice
422 TransferSettings vol_settings(_settings);
423
424 // add custom headers for download.opensuse.org (bsc#955801)
425 if ( _url.getHost() == "download.opensuse.org" )
426 {
427 vol_settings.addHeader(anonymousIdHeader());
428 vol_settings.addHeader(distributionFlavorHeader());
429 }
430 vol_settings.addHeader("Pragma:");
431
433
434 // fill some settings from url query parameters
435 try
436 {
438 }
439 catch ( const MediaException &e )
440 {
442 ZYPP_RETHROW(e);
443 }
444 // if the proxy was not set (or explicitly unset) by url, then look...
445 if ( _settings.proxy().empty() )
446 {
447 // ...at the system proxy settings
449 }
450
453 {
454 case 4: SET_OPTION(CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); break;
455 case 6: SET_OPTION(CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6); break;
456 }
457
461 SET_OPTION(CURLOPT_CONNECTTIMEOUT, _settings.connectTimeout());
462 // If a transfer timeout is set, also set CURLOPT_TIMEOUT to an upper limit
463 // just in case curl does not trigger its progress callback frequently
464 // enough.
465 if ( _settings.timeout() )
466 {
467 SET_OPTION(CURLOPT_TIMEOUT, 3600L);
468 }
469
470 // follow any Location: header that the server sends as part of
471 // an HTTP header (#113275)
472 SET_OPTION(CURLOPT_FOLLOWLOCATION, 1L);
473 // 3 redirects seem to be too few in some cases (bnc #465532)
474 SET_OPTION(CURLOPT_MAXREDIRS, 6L);
475
476 if ( _url.getScheme() == "https" )
477 {
478#if CURLVERSION_AT_LEAST(7,19,4)
479 // restrict following of redirections from https to https only
480 if ( _url.getHost() == "download.opensuse.org" )
481 SET_OPTION( CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS );
482 else
483 SET_OPTION( CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTPS );
484#endif
485
488 {
490 }
491
493 {
494 SET_OPTION(CURLOPT_SSLCERT, _settings.clientCertificatePath().c_str());
495 }
496 if( ! _settings.clientKeyPath().empty() )
497 {
498 SET_OPTION(CURLOPT_SSLKEY, _settings.clientKeyPath().c_str());
499 }
500
501#ifdef CURLSSLOPT_ALLOW_BEAST
502 // see bnc#779177
503 ret = curl_easy_setopt( _curl, CURLOPT_SSL_OPTIONS, CURLSSLOPT_ALLOW_BEAST );
504 if ( ret != 0 ) {
507 }
508#endif
509 SET_OPTION(CURLOPT_SSL_VERIFYPEER, _settings.verifyPeerEnabled() ? 1L : 0L);
510 SET_OPTION(CURLOPT_SSL_VERIFYHOST, _settings.verifyHostEnabled() ? 2L : 0L);
511 // bnc#903405 - POODLE: libzypp should only talk TLS
512 SET_OPTION(CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1);
513 }
514
515 SET_OPTION(CURLOPT_USERAGENT, _settings.userAgentString().c_str() );
516
517 /* Fixes bsc#1174011 "auth=basic ignored in some cases"
518 * We should proactively add the password to the request if basic auth is configured
519 * and a password is available in the credentials but not in the URL.
520 *
521 * We will be a bit paranoid here and require that the URL has a user embedded, otherwise we go the default route
522 * and ask the server first about the auth method
523 */
524 if ( _settings.authType() == "basic"
525 && _settings.username().size()
526 && !_settings.password().size() ) {
527
529 const auto cred = cm.getCred( _url );
530 if ( cred && cred->valid() ) {
531 if ( !_settings.username().size() )
532 _settings.setUsername(cred->username());
533 _settings.setPassword(cred->password());
534 }
535 }
536
537 /*---------------------------------------------------------------*
538 CURLOPT_USERPWD: [user name]:[password]
539
540 Url::username/password -> CURLOPT_USERPWD
541 If not provided, anonymous FTP identification
542 *---------------------------------------------------------------*/
543
544 if ( _settings.userPassword().size() )
545 {
546 SET_OPTION(CURLOPT_USERPWD, _settings.userPassword().c_str());
547 std::string use_auth = _settings.authType();
548 if (use_auth.empty())
549 use_auth = "digest,basic"; // our default
550 long auth = CurlAuthData::auth_type_str2long(use_auth);
551 if( auth != CURLAUTH_NONE)
552 {
553 DBG << "Enabling HTTP authentication methods: " << use_auth
554 << " (CURLOPT_HTTPAUTH=" << auth << ")" << std::endl;
555 SET_OPTION(CURLOPT_HTTPAUTH, auth);
556 }
557 }
558
559 if ( _settings.proxyEnabled() && ! _settings.proxy().empty() )
560 {
561 DBG << "Proxy: '" << _settings.proxy() << "'" << endl;
562 SET_OPTION(CURLOPT_PROXY, _settings.proxy().c_str());
563 SET_OPTION(CURLOPT_PROXYAUTH, CURLAUTH_BASIC|CURLAUTH_DIGEST|CURLAUTH_NTLM );
564 /*---------------------------------------------------------------*
565 * CURLOPT_PROXYUSERPWD: [user name]:[password]
566 *
567 * Url::option(proxyuser and proxypassword) -> CURLOPT_PROXYUSERPWD
568 * If not provided, $HOME/.curlrc is evaluated
569 *---------------------------------------------------------------*/
570
571 std::string proxyuserpwd = _settings.proxyUserPassword();
572
573 if ( proxyuserpwd.empty() )
574 {
575 CurlConfig curlconf;
576 CurlConfig::parseConfig(curlconf); // parse ~/.curlrc
577 if ( curlconf.proxyuserpwd.empty() )
578 DBG << "Proxy: ~/.curlrc does not contain the proxy-user option" << endl;
579 else
580 {
581 proxyuserpwd = curlconf.proxyuserpwd;
582 DBG << "Proxy: using proxy-user from ~/.curlrc" << endl;
583 }
584 }
585 else
586 {
587 DBG << "Proxy: using provided proxy-user '" << _settings.proxyUsername() << "'" << endl;
588 }
589
590 if ( ! proxyuserpwd.empty() )
591 {
592 SET_OPTION(CURLOPT_PROXYUSERPWD, curlUnEscape( proxyuserpwd ).c_str());
593 }
594 }
595#if CURLVERSION_AT_LEAST(7,19,4)
596 else if ( _settings.proxy() == EXPLICITLY_NO_PROXY )
597 {
598 // Explicitly disabled in URL (see fillSettingsFromUrl()).
599 // This should also prevent libcurl from looking into the environment.
600 DBG << "Proxy: explicitly NOPROXY" << endl;
601 SET_OPTION(CURLOPT_NOPROXY, "*");
602 }
603#endif
604 else
605 {
606 DBG << "Proxy: not explicitly set" << endl;
607 DBG << "Proxy: libcurl may look into the environment" << endl;
608 }
609
611 if ( _settings.minDownloadSpeed() != 0 )
612 {
613 SET_OPTION(CURLOPT_LOW_SPEED_LIMIT, _settings.minDownloadSpeed());
614 // default to 10 seconds at low speed
615 SET_OPTION(CURLOPT_LOW_SPEED_TIME, 60L);
616 }
617
618#if CURLVERSION_AT_LEAST(7,15,5)
619 if ( _settings.maxDownloadSpeed() != 0 )
620 SET_OPTION_OFFT(CURLOPT_MAX_RECV_SPEED_LARGE, _settings.maxDownloadSpeed());
621#endif
622
623 /*---------------------------------------------------------------*
624 *---------------------------------------------------------------*/
625
628 if ( str::strToBool( _url.getQueryParam( "cookies" ), true ) )
629 SET_OPTION(CURLOPT_COOKIEFILE, _currentCookieFile.c_str() );
630 else
631 MIL << "No cookies requested" << endl;
632 SET_OPTION(CURLOPT_COOKIEJAR, _currentCookieFile.c_str() );
633 SET_OPTION(CURLOPT_PROGRESSFUNCTION, &progressCallback );
634 SET_OPTION(CURLOPT_NOPROGRESS, 0L);
635
636#if CURLVERSION_AT_LEAST(7,18,0)
637 // bnc #306272
638 SET_OPTION(CURLOPT_PROXY_TRANSFER_MODE, 1L );
639#endif
640 // Append settings custom headers to curl.
641 // TransferSettings assert strings are trimmed (HTTP/2 RFC 9113)
642 for ( const auto &header : vol_settings.headers() ) {
643 _customHeaders = curl_slist_append(_customHeaders, header.c_str());
644 if ( !_customHeaders )
646 }
647 SET_OPTION(CURLOPT_HTTPHEADER, _customHeaders);
648}
649
651
652
653void MediaCurl::attachTo (bool next)
654{
655 if ( next )
657
658 if ( !_url.isValid() )
660
663 {
665 }
666
667 disconnectFrom(); // clean _curl if needed
668 _curl = curl_easy_init();
669 if ( !_curl ) {
671 }
672 try
673 {
674 setupEasy();
675 }
676 catch (Exception & ex)
677 {
679 ZYPP_RETHROW(ex);
680 }
681
682 // FIXME: need a derived class to propelly compare url's
684 setMediaSource(media);
685}
686
687bool
689{
690 return MediaHandler::checkAttachPoint( apoint, true, true);
691}
692
694
696{
697 if ( _customHeaders )
698 {
699 curl_slist_free_all(_customHeaders);
700 _customHeaders = 0L;
701 }
702
703 if ( _curl )
704 {
705 // bsc#1201092: Strange but within global_dtors we may exceptions here.
706 try { curl_easy_cleanup( _curl ); }
707 catch (...) { ; }
708 _curl = NULL;
709 }
710}
711
713
714void MediaCurl::releaseFrom( const std::string & ejectDev )
715{
716 disconnect();
717}
718
720
721void MediaCurl::getFile( const OnMediaLocation &file ) const
722{
723 // Use absolute file name to prevent access of files outside of the
724 // hierarchy below the attach point.
725 getFileCopy( file, localPath(file.filename()).absolutename() );
726}
727
729
730void MediaCurl::getFileCopy( const OnMediaLocation & srcFile , const Pathname & target ) const
731{
732
733 const auto &filename = srcFile.filename();
734
735 // Optional files will send no report until data are actually received (we know it exists).
736 OptionalDownloadProgressReport reportfilter( srcFile.optional() );
738
739 Url fileurl(getFileUrl(filename));
740
741 bool firstAuth = true; // bsc#1210870: authenticate must not return stored credentials more than once.
742 unsigned internalTry = 0;
743 static constexpr unsigned maxInternalTry = 3;
744
745 do
746 {
747 try
748 {
749 doGetFileCopy( srcFile, target, report );
750 break; // success!
751 }
752 // retry with proper authentication data
753 catch (MediaUnauthorizedException & ex_r)
754 {
755 if ( authenticate(ex_r.hint(), firstAuth) ) {
756 firstAuth = false; // must not return stored credentials again
757 continue; // retry
758 }
759
761 ZYPP_RETHROW(ex_r);
762 }
763 // unexpected exception
764 catch (MediaException & excpt_r)
765 {
766 if ( typeid(excpt_r) == typeid( MediaCurlExceptionMayRetryInternaly ) ) {
767 ++internalTry;
768 if ( internalTry < maxInternalTry ) {
769 // just report (NO_ERROR); no interactive request to the user
770 report->problem(fileurl, media::DownloadProgressReport::NO_ERROR, excpt_r.asUserHistory()+_("Will try again..."));
771 continue; // retry
772 }
773 excpt_r.addHistory( str::Format(_("Giving up after %1% attempts.")) % maxInternalTry );
774 }
775
777 if( typeid(excpt_r) == typeid( media::MediaFileNotFoundException ) ||
778 typeid(excpt_r) == typeid( media::MediaNotAFileException ) )
779 {
781 }
782 report->finish(fileurl, reason, excpt_r.asUserHistory());
783 ZYPP_RETHROW(excpt_r);
784 }
785 }
786 while ( true );
787 report->finish(fileurl, zypp::media::DownloadProgressReport::NO_ERROR, "");
788}
789
791
792bool MediaCurl::getDoesFileExist( const Pathname & filename ) const
793{
794 bool retry = false;
795
796 do
797 {
798 try
799 {
800 return doGetDoesFileExist( filename );
801 }
802 // authentication problem, retry with proper authentication data
803 catch (MediaUnauthorizedException & ex_r)
804 {
805 if(authenticate(ex_r.hint(), !retry))
806 retry = true;
807 else
808 ZYPP_RETHROW(ex_r);
809 }
810 // unexpected exception
811 catch (MediaException & excpt_r)
812 {
813 ZYPP_RETHROW(excpt_r);
814 }
815 }
816 while (retry);
817
818 return false;
819}
820
822
824 CURLcode code,
825 bool timeout_reached) const
826{
827 if ( code != 0 )
828 {
829 Url url;
830 if (filename.empty())
831 url = _url;
832 else
833 url = getFileUrl(filename);
834
835 std::string err;
836 {
837 switch ( code )
838 {
839 case CURLE_UNSUPPORTED_PROTOCOL:
840 err = " Unsupported protocol";
841 if ( !_lastRedirect.empty() )
842 {
843 err += " or redirect (";
844 err += _lastRedirect;
845 err += ")";
846 }
847 break;
848 case CURLE_URL_MALFORMAT:
849 case CURLE_URL_MALFORMAT_USER:
850 err = " Bad URL";
851 break;
852 case CURLE_LOGIN_DENIED:
854 MediaUnauthorizedException(url, "Login failed.", _curlError, ""));
855 break;
856 case CURLE_HTTP_RETURNED_ERROR:
857 {
858 long httpReturnCode = 0;
859 CURLcode infoRet = curl_easy_getinfo( _curl,
860 CURLINFO_RESPONSE_CODE,
861 &httpReturnCode );
862 if ( infoRet == CURLE_OK )
863 {
864 std::string msg = "HTTP response: " + str::numstring( httpReturnCode );
865 switch ( httpReturnCode )
866 {
867 case 401:
868 {
869 std::string auth_hint = getAuthHint();
870
871 DBG << msg << " Login failed (URL: " << url.asString() << ")" << std::endl;
872 DBG << "MediaUnauthorizedException auth hint: '" << auth_hint << "'" << std::endl;
873
875 url, "Login failed.", _curlError, auth_hint
876 ));
877 }
878
879 case 502: // bad gateway (bnc #1070851)
880 case 503: // service temporarily unavailable (bnc #462545)
882 case 504: // gateway timeout
884 case 403:
885 {
886 std::string msg403;
887 if ( url.getHost().find(".suse.com") != std::string::npos )
888 msg403 = _("Visit the SUSE Customer Center to check whether your registration is valid and has not expired.");
889 else if (url.asString().find("novell.com") != std::string::npos)
890 msg403 = _("Visit the Novell Customer Center to check whether your registration is valid and has not expired.");
892 }
893 case 404:
894 case 410:
896 }
897
898 DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
900 }
901 else
902 {
903 std::string msg = "Unable to retrieve HTTP response:";
904 DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
906 }
907 }
908 break;
909 case CURLE_FTP_COULDNT_RETR_FILE:
910#if CURLVERSION_AT_LEAST(7,16,0)
911 case CURLE_REMOTE_FILE_NOT_FOUND:
912#endif
913 case CURLE_FTP_ACCESS_DENIED:
914 case CURLE_TFTP_NOTFOUND:
915 err = "File not found";
917 break;
918 case CURLE_BAD_PASSWORD_ENTERED:
919 case CURLE_FTP_USER_PASSWORD_INCORRECT:
920 err = "Login failed";
921 break;
922 case CURLE_COULDNT_RESOLVE_PROXY:
923 case CURLE_COULDNT_RESOLVE_HOST:
924 case CURLE_COULDNT_CONNECT:
925 case CURLE_FTP_CANT_GET_HOST:
926 err = "Connection failed";
927 break;
928 case CURLE_WRITE_ERROR:
929 err = "Write error";
930 break;
931 case CURLE_PARTIAL_FILE:
932 case CURLE_OPERATION_TIMEDOUT:
933 timeout_reached = true; // fall though to TimeoutException
934 // fall though...
935 case CURLE_ABORTED_BY_CALLBACK:
936 if( timeout_reached )
937 {
938 err = "Timeout reached";
940 }
941 else
942 {
943 err = "User abort";
944 }
945 break;
946
947 // Attempt to work around certain issues by autoretry in MediaCurl::getFileCopy
948 case CURLE_HTTP2:
949 case CURLE_HTTP2_STREAM:
950 err = "Curl error " + str::numstring( code );
952 break;
953
954 default:
955 err = "Curl error " + str::numstring( code );
956 break;
957 }
958
959 // uhm, no 0 code but unknown curl exception
961 }
962 }
963 else
964 {
965 // actually the code is 0, nothing happened
966 }
967}
968
970
971bool MediaCurl::doGetDoesFileExist( const Pathname & filename ) const
972{
973 DBG << filename.asString() << endl;
974
975 if(!_url.isValid())
977
978 if(_url.getHost().empty())
980
981 Url url(getFileUrl(filename));
982
983 DBG << "URL: " << url.asString() << endl;
984 // Use URL without options and without username and passwd
985 // (some proxies dislike them in the URL).
986 // Curl seems to need the just scheme, hostname and a path;
987 // the rest was already passed as curl options (in attachTo).
988 Url curlUrl( clearQueryString(url) );
989
990 //
991 // See also Bug #154197 and ftp url definition in RFC 1738:
992 // The url "ftp://user@host/foo/bar/file" contains a path,
993 // that is relative to the user's home.
994 // The url "ftp://user@host//foo/bar/file" (or also with
995 // encoded slash as %2f) "ftp://user@host/%2ffoo/bar/file"
996 // contains an absolute path.
997 //
998 _lastRedirect.clear();
999 std::string urlBuffer( curlUrl.asString());
1000 CURLcode ret = curl_easy_setopt( _curl, CURLOPT_URL,
1001 urlBuffer.c_str() );
1002 if ( ret != 0 ) {
1004 }
1005
1006 // If no head requests allowed (?head_requests=no):
1007 // Instead of returning no data with NOBODY, we return
1008 // little data, that works with broken servers, and
1009 // works for ftp as well, because retrieving only headers
1010 // ftp will return always OK code ?
1011 // See http://curl.haxx.se/docs/knownbugs.html #58
1013 struct TempSetHeadRequest
1014 {
1015 TempSetHeadRequest( CURL * curl_r, bool doHttpHeadRequest_r )
1016 : _curl { curl_r }
1017 , _doHttpHeadRequest { doHttpHeadRequest_r }
1018 {
1019 if ( _doHttpHeadRequest ) {
1020 curl_easy_setopt( _curl, CURLOPT_NOBODY, 1L );
1021 } else {
1022 curl_easy_setopt( _curl, CURLOPT_RANGE, "0-1" );
1023 }
1024 }
1025 ~TempSetHeadRequest() {
1026 if ( _doHttpHeadRequest ) {
1027 curl_easy_setopt( _curl, CURLOPT_NOBODY, 0L);
1028 /* yes, this is why we never got to get NOBODY working before,
1029 because setting it changes this option too, and we also*
1030 need to reset it
1031 See: http://curl.haxx.se/mail/archive-2005-07/0073.html
1032 */
1033 curl_easy_setopt( _curl, CURLOPT_HTTPGET, 1L );
1034 } else {
1035 curl_easy_setopt( _curl, CURLOPT_RANGE, NULL );
1036 }
1037 }
1038 private:
1039 CURL * _curl;
1040 bool _doHttpHeadRequest;
1041 } _guard( _curl, (_url.getScheme() == "http" || _url.getScheme() == "https") && _settings.headRequestsAllowed() );
1042
1043
1044 AutoFILE file { ::fopen( "/dev/null", "w" ) };
1045 if ( !file ) {
1046 ERR << "fopen failed for /dev/null" << endl;
1047 ZYPP_THROW(MediaWriteException("/dev/null"));
1048 }
1049
1050 ret = curl_easy_setopt( _curl, CURLOPT_WRITEDATA, (*file) );
1051 if ( ret != 0 ) {
1053 }
1054
1055 CURLcode ok = curl_easy_perform( _curl );
1056 MIL << "perform code: " << ok << " [ " << curl_easy_strerror(ok) << " ]" << endl;
1057
1058 // as we are not having user interaction, the user can't cancel
1059 // the file existence checking, a callback or timeout return code
1060 // will be always a timeout.
1061 try {
1062 evaluateCurlCode( filename, ok, true /* timeout */);
1063 }
1064 catch ( const MediaFileNotFoundException &e ) {
1065 // if the file did not exist then we can return false
1066 return false;
1067 }
1068 catch ( const MediaException &e ) {
1069 // some error, we are not sure about file existence, rethrw
1070 ZYPP_RETHROW(e);
1071 }
1072 // exists
1073 return ( ok == CURLE_OK );
1074}
1075
1077
1078void MediaCurl::doGetFileCopy( const OnMediaLocation &srcFile , const Pathname & target, callback::SendReport<DownloadProgressReport> & report, RequestOptions options ) const
1079{
1080 Pathname dest = target.absolutename();
1081 if( assert_dir( dest.dirname() ) )
1082 {
1083 DBG << "assert_dir " << dest.dirname() << " failed" << endl;
1084 ZYPP_THROW( MediaSystemException(getFileUrl(srcFile.filename()), "System error on " + dest.dirname().asString()) );
1085 }
1086
1087 ManagedFile destNew { target.extend( ".new.zypp.XXXXXX" ) };
1088 AutoFILE file;
1089 {
1090 AutoFREE<char> buf { ::strdup( (*destNew).c_str() ) };
1091 if( ! buf )
1092 {
1093 ERR << "out of memory for temp file name" << endl;
1094 ZYPP_THROW(MediaSystemException(getFileUrl(srcFile.filename()), "out of memory for temp file name"));
1095 }
1096
1097 AutoFD tmp_fd { ::mkostemp( buf, O_CLOEXEC ) };
1098 if( tmp_fd == -1 )
1099 {
1100 ERR << "mkstemp failed for file '" << destNew << "'" << endl;
1102 }
1103 destNew = ManagedFile( (*buf), filesystem::unlink );
1104
1105 file = ::fdopen( tmp_fd, "we" );
1106 if ( ! file )
1107 {
1108 ERR << "fopen failed for file '" << destNew << "'" << endl;
1110 }
1111 tmp_fd.resetDispose(); // don't close it here! ::fdopen moved ownership to file
1112 }
1113
1114 DBG << "dest: " << dest << endl;
1115 DBG << "temp: " << destNew << endl;
1116
1117 // set IFMODSINCE time condition (no download if not modified)
1118 if( PathInfo(target).isExist() && !(options & OPTION_NO_IFMODSINCE) )
1119 {
1120 curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
1121 curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, (long)PathInfo(target).mtime());
1122 }
1123 else
1124 {
1125 curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1126 curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1127 }
1128 try
1129 {
1130 doGetFileCopyFile( srcFile, dest, file, report, options);
1131 }
1132 catch (Exception &e)
1133 {
1134 curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1135 curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1136 ZYPP_RETHROW(e);
1137 }
1138
1139 long httpReturnCode = 0;
1140 CURLcode infoRet = curl_easy_getinfo(_curl,
1141 CURLINFO_RESPONSE_CODE,
1142 &httpReturnCode);
1143 bool modified = true;
1144 if (infoRet == CURLE_OK)
1145 {
1146 DBG << "HTTP response: " + str::numstring(httpReturnCode);
1147 if ( httpReturnCode == 304
1148 || ( httpReturnCode == 213 && (_url.getScheme() == "ftp" || _url.getScheme() == "tftp") ) ) // not modified
1149 {
1150 DBG << " Not modified.";
1151 modified = false;
1152 }
1153 DBG << endl;
1154 }
1155 else
1156 {
1157 WAR << "Could not get the response code." << endl;
1158 }
1159
1160 if (modified || infoRet != CURLE_OK)
1161 {
1162 // apply umask
1163 if ( ::fchmod( ::fileno(file), filesystem::applyUmaskTo( 0644 ) ) )
1164 {
1165 ERR << "Failed to chmod file " << destNew << endl;
1166 }
1167
1168 file.resetDispose(); // we're going to close it manually here
1169 if ( ::fclose( file ) )
1170 {
1171 ERR << "Fclose failed for file '" << destNew << "'" << endl;
1173 }
1174
1175 // move the temp file into dest
1176 if ( rename( destNew, dest ) != 0 ) {
1177 ERR << "Rename failed" << endl;
1179 }
1180 destNew.resetDispose(); // no more need to unlink it
1181 }
1182
1183 DBG << "done: " << PathInfo(dest) << endl;
1184}
1185
1187
1188void MediaCurl::doGetFileCopyFile( const OnMediaLocation & srcFile, const Pathname & dest, FILE *file, callback::SendReport<DownloadProgressReport> & report, RequestOptions options ) const
1189{
1190 DBG << srcFile.filename().asString() << endl;
1191
1192 if(!_url.isValid())
1194
1195 if(_url.getHost().empty())
1197
1198 Url url(getFileUrl(srcFile.filename()));
1199
1200 DBG << "URL: " << url.asString() << endl;
1201 // Use URL without options and without username and passwd
1202 // (some proxies dislike them in the URL).
1203 // Curl seems to need the just scheme, hostname and a path;
1204 // the rest was already passed as curl options (in attachTo).
1205 Url curlUrl( clearQueryString(url) );
1206
1207 //
1208 // See also Bug #154197 and ftp url definition in RFC 1738:
1209 // The url "ftp://user@host/foo/bar/file" contains a path,
1210 // that is relative to the user's home.
1211 // The url "ftp://user@host//foo/bar/file" (or also with
1212 // encoded slash as %2f) "ftp://user@host/%2ffoo/bar/file"
1213 // contains an absolute path.
1214 //
1215 _lastRedirect.clear();
1216 std::string urlBuffer( curlUrl.asString());
1217 CURLcode ret = curl_easy_setopt( _curl, CURLOPT_URL,
1218 urlBuffer.c_str() );
1219 if ( ret != 0 ) {
1221 }
1222
1223 ret = curl_easy_setopt( _curl, CURLOPT_WRITEDATA, file );
1224 if ( ret != 0 ) {
1226 }
1227
1228 // Set callback and perform.
1229 internal::ProgressData progressData(_curl, _settings.timeout(), url, srcFile.downloadSize(), &report);
1230 if (!(options & OPTION_NO_REPORT_START))
1231 report->start(url, dest);
1232 if ( curl_easy_setopt( _curl, CURLOPT_PROGRESSDATA, &progressData ) != 0 ) {
1233 WAR << "Can't set CURLOPT_PROGRESSDATA: " << _curlError << endl;;
1234 }
1235
1236 ret = curl_easy_perform( _curl );
1237#if CURLVERSION_AT_LEAST(7,19,4)
1238 // bnc#692260: If the client sends a request with an If-Modified-Since header
1239 // with a future date for the server, the server may respond 200 sending a
1240 // zero size file.
1241 // curl-7.19.4 introduces CURLINFO_CONDITION_UNMET to check this condition.
1242 if ( ftell(file) == 0 && ret == 0 )
1243 {
1244 long httpReturnCode = 33;
1245 if ( curl_easy_getinfo( _curl, CURLINFO_RESPONSE_CODE, &httpReturnCode ) == CURLE_OK && httpReturnCode == 200 )
1246 {
1247 long conditionUnmet = 33;
1248 if ( curl_easy_getinfo( _curl, CURLINFO_CONDITION_UNMET, &conditionUnmet ) == CURLE_OK && conditionUnmet )
1249 {
1250 WAR << "TIMECONDITION unmet - retry without." << endl;
1251 curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1252 curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1253 ret = curl_easy_perform( _curl );
1254 }
1255 }
1256 }
1257#endif
1258
1259 if ( curl_easy_setopt( _curl, CURLOPT_PROGRESSDATA, NULL ) != 0 ) {
1260 WAR << "Can't unset CURLOPT_PROGRESSDATA: " << _curlError << endl;;
1261 }
1262
1263 if ( ret != 0 )
1264 {
1265 ERR << "curl error: " << ret << ": " << _curlError
1266 << ", temp file size " << ftell(file)
1267 << " bytes." << endl;
1268
1269 // the timeout is determined by the progress data object
1270 // which holds whether the timeout was reached or not,
1271 // otherwise it would be a user cancel
1272 try {
1273
1274 if ( progressData.fileSizeExceeded() )
1276
1277 evaluateCurlCode( srcFile.filename(), ret, progressData.timeoutReached() );
1278 }
1279 catch ( const MediaException &e ) {
1280 // some error, we are not sure about file existence, rethrw
1281 ZYPP_RETHROW(e);
1282 }
1283 }
1284}
1285
1287
1288void MediaCurl::getDir( const Pathname & dirname, bool recurse_r ) const
1289{
1290 filesystem::DirContent content;
1291 getDirInfo( content, dirname, /*dots*/false );
1292
1293 for ( filesystem::DirContent::const_iterator it = content.begin(); it != content.end(); ++it ) {
1294 Pathname filename = dirname + it->name;
1295 int res = 0;
1296
1297 switch ( it->type ) {
1298 case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all
1300 getFile( OnMediaLocation( filename ) );
1301 break;
1302 case filesystem::FT_DIR: // newer directory.yast contain at least directory info
1303 if ( recurse_r ) {
1304 getDir( filename, recurse_r );
1305 } else {
1306 res = assert_dir( localPath( filename ) );
1307 if ( res ) {
1308 WAR << "Ignore error (" << res << ") on creating local directory '" << localPath( filename ) << "'" << endl;
1309 }
1310 }
1311 break;
1312 default:
1313 // don't provide devices, sockets, etc.
1314 break;
1315 }
1316 }
1317}
1318
1320
1321void MediaCurl::getDirInfo( std::list<std::string> & retlist,
1322 const Pathname & dirname, bool dots ) const
1323{
1324 getDirectoryYast( retlist, dirname, dots );
1325}
1326
1328
1330 const Pathname & dirname, bool dots ) const
1331{
1332 getDirectoryYast( retlist, dirname, dots );
1333}
1334
1336//
1337int MediaCurl::aliveCallback( void *clientp, double /*dltotal*/, double dlnow, double /*ultotal*/, double /*ulnow*/ )
1338{
1339 internal::ProgressData *pdata = reinterpret_cast<internal::ProgressData *>( clientp );
1340 if( pdata )
1341 {
1342 // Do not propagate dltotal in alive callbacks. MultiCurl uses this to
1343 // prevent a percentage raise while downloading a metalink file. Download
1344 // activity however is indicated by propagating the download rate (via dlnow).
1345 pdata->updateStats( 0.0, dlnow );
1346 return pdata->reportProgress();
1347 }
1348 return 0;
1349}
1350
1351int MediaCurl::progressCallback( void *clientp, double dltotal, double dlnow, double ultotal, double ulnow )
1352{
1353 internal::ProgressData *pdata = reinterpret_cast<internal::ProgressData *>( clientp );
1354 if( pdata )
1355 {
1356 // work around curl bug that gives us old data
1357 long httpReturnCode = 0;
1358 if ( curl_easy_getinfo( pdata->curl(), CURLINFO_RESPONSE_CODE, &httpReturnCode ) != CURLE_OK || httpReturnCode == 0 )
1359 return aliveCallback( clientp, dltotal, dlnow, ultotal, ulnow );
1360
1361 pdata->updateStats( dltotal, dlnow );
1362 return pdata->reportProgress();
1363 }
1364 return 0;
1365}
1366
1368{
1369 internal::ProgressData *pdata = reinterpret_cast<internal::ProgressData *>(clientp);
1370 return pdata ? pdata->curl() : 0;
1371}
1372
1374
1375std::string MediaCurl::getAuthHint() const
1376{
1377 long auth_info = CURLAUTH_NONE;
1378
1379 CURLcode infoRet =
1380 curl_easy_getinfo(_curl, CURLINFO_HTTPAUTH_AVAIL, &auth_info);
1381
1382 if(infoRet == CURLE_OK)
1383 {
1384 return CurlAuthData::auth_type_long2str(auth_info);
1385 }
1386
1387 return "";
1388}
1389
1394void MediaCurl::resetExpectedFileSize(void *clientp, const ByteCount &expectedFileSize)
1395{
1396 internal::ProgressData *data = reinterpret_cast<internal::ProgressData *>(clientp);
1397 if ( data ) {
1398 data->expectedFileSize( expectedFileSize );
1399 }
1400}
1401
1403
1404bool MediaCurl::authenticate(const std::string & availAuthTypes, bool firstTry) const
1405{
1408 CurlAuthData_Ptr credentials;
1409
1410 // get stored credentials
1411 AuthData_Ptr cmcred = cm.getCred(_url);
1412
1413 if (cmcred && firstTry)
1414 {
1415 credentials.reset(new CurlAuthData(*cmcred));
1416 DBG << "got stored credentials:" << endl << *credentials << endl;
1417 }
1418 // if not found, ask user
1419 else
1420 {
1421
1422 CurlAuthData_Ptr curlcred;
1423 curlcred.reset(new CurlAuthData());
1425
1426 // preset the username if present in current url
1427 if (!_url.getUsername().empty() && firstTry)
1428 curlcred->setUsername(_url.getUsername());
1429 // if CM has found some credentials, preset the username from there
1430 else if (cmcred)
1431 curlcred->setUsername(cmcred->username());
1432
1433 // indicate we have no good credentials from CM
1434 cmcred.reset();
1435
1436 std::string prompt_msg = str::Format(_("Authentication required for '%s'")) % _url.asString();
1437
1438 // set available authentication types from the exception
1439 // might be needed in prompt
1440 curlcred->setAuthType(availAuthTypes);
1441
1442 // ask user
1443 if (auth_report->prompt(_url, prompt_msg, *curlcred))
1444 {
1445 DBG << "callback answer: retry" << endl
1446 << "CurlAuthData: " << *curlcred << endl;
1447
1448 if (curlcred->valid())
1449 {
1450 credentials = curlcred;
1451 // if (credentials->username() != _url.getUsername())
1452 // _url.setUsername(credentials->username());
1460 }
1461 }
1462 else
1463 {
1464 DBG << "callback answer: cancel" << endl;
1465 }
1466 }
1467
1468 // set username and password
1469 if (credentials)
1470 {
1471 // HACK, why is this const?
1472 const_cast<MediaCurl*>(this)->_settings.setUsername(credentials->username());
1473 const_cast<MediaCurl*>(this)->_settings.setPassword(credentials->password());
1474
1475 // set username and password
1476 CURLcode ret = curl_easy_setopt(_curl, CURLOPT_USERPWD, _settings.userPassword().c_str());
1478
1479 // set available authentication types from the exception
1480 if (credentials->authType() == CURLAUTH_NONE)
1481 credentials->setAuthType(availAuthTypes);
1482
1483 // set auth type (seems this must be set _after_ setting the userpwd)
1484 if (credentials->authType() != CURLAUTH_NONE)
1485 {
1486 // FIXME: only overwrite if not empty?
1487 const_cast<MediaCurl*>(this)->_settings.setAuthType(credentials->authTypeAsString());
1488 ret = curl_easy_setopt(_curl, CURLOPT_HTTPAUTH, credentials->authType());
1490 }
1491
1492 if (!cmcred)
1493 {
1494 credentials->setUrl(_url);
1495 cm.addCred(*credentials);
1496 cm.save();
1497 }
1498
1499 return true;
1500 }
1501
1502 return false;
1503}
1504
1505//need a out of line definiton, otherwise vtable is emitted for every translation unit
1507
1508 } // namespace media
1509} // namespace zypp
1510//
#define LIBZYPP_VERSION_STRING
Definition: APIConfig.h:15
#define SET_OPTION_OFFT(opt, val)
Definition: MediaCurl.cc:330
#define SET_OPTION(opt, val)
Definition: MediaCurl.cc:323
Edition * _value
Definition: SysContent.cc:311
TrueBool _guard
Definition: TargetImpl.cc:1596
Attempt to work around certain issues by autoretry in MediaCurl::getFileCopy E.g.
Definition: MediaCurl.cc:298
MediaCurlExceptionMayRetryInternaly(const Url &url_r, const std::string &err_r, const std::string &msg_r)
Definition: MediaCurl.cc:300
void resetDispose()
Set no dispose function.
Definition: AutoDispose.h:180
Store and operate with byte count.
Definition: ByteCount.h:31
Base class for Exception.
Definition: Exception.h:146
std::string asUserHistory() const
A single (multiline) string composed of asUserString and historyAsString.
Definition: Exception.cc:91
void addHistory(const std::string &msg_r)
Add some message text to the history.
Definition: Exception.cc:140
const char * c_str() const
Definition: IdStringType.h:105
Describes a resource file located on a medium.
bool optional() const
Whether this is an optional resource.
const ByteCount & downloadSize() const
The size of the resource on the server.
const Pathname & filename() const
The path to the resource on the medium.
ProgressData()
Ctor no range [0,0](0).
Definition: progressdata.h:174
std::string distributionFlavor() const
This is flavor attribute of the installed base product but does not require the target to be loaded a...
Definition: Target.cc:127
std::string anonymousUniqueId() const
anonymous unique id
Definition: Target.cc:132
std::string targetDistribution() const
This is register.target attribute of the installed base product.
Definition: Target.cc:102
Url manipulation class.
Definition: Url.h:92
std::string getScheme() const
Returns the scheme name of the URL.
Definition: Url.cc:533
std::string asString() const
Returns a default string representation of the Url object.
Definition: Url.cc:497
std::string getUsername(EEncoding eflag=zypp::url::E_DECODED) const
Returns the username from the URL authority.
Definition: Url.cc:572
std::string getQueryParam(const std::string &param, EEncoding eflag=zypp::url::E_DECODED) const
Return the value for the specified query parameter.
Definition: Url.cc:660
std::string getHost(EEncoding eflag=zypp::url::E_DECODED) const
Returns the hostname or IP from the URL authority.
Definition: Url.cc:588
bool isValid() const
Verifies the Url.
Definition: Url.cc:489
static ZConfig & instance()
Singleton ctor.
Definition: ZConfig.cc:922
Typesafe passing of user data via callbacks.
Definition: UserData.h:39
Wrapper class for stat/lstat.
Definition: PathInfo.h:221
bool userMayRWX() const
Definition: PathInfo.h:353
const Pathname & path() const
Return current Pathname.
Definition: PathInfo.h:246
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
Definition: Pathname.h:173
Pathname dirname() const
Return all but the last component od this path.
Definition: Pathname.h:124
const char * c_str() const
String representation.
Definition: Pathname.h:110
const std::string & asString() const
String representation.
Definition: Pathname.h:91
bool empty() const
Test for an empty path.
Definition: Pathname.h:114
Pathname absolutename() const
Return this path, adding a leading '/' if relative.
Definition: Pathname.h:139
void save()
Saves any unsaved credentials added via addUserCred() or addGlobalCred() methods.
AuthData_Ptr getCred(const Url &url)
Get credentials for the specified url.
void addCred(const AuthData &cred)
Add new credentials with user callbacks.
Curl HTTP authentication data.
Definition: curlauthdata.h:22
static std::string auth_type_long2str(long auth_type)
Converts a long of ORed CURLAUTH_* identifiers into a string of comma separated list of authenticatio...
static long auth_type_str2long(std::string &auth_type_str)
Converts a string of comma separated list of authetication type names into a long of ORed CURLAUTH_* ...
Definition: curlauthdata.cc:50
MediaCurlException(const Url &url_r, const std::string &err_r, const std::string &msg_r)
Implementation class for FTP, HTTP and HTTPS MediaHandler.
Definition: MediaCurl.h:32
virtual void setupEasy()
initializes the curl easy handle with the data from the url
Definition: MediaCurl.cc:406
static void setCookieFile(const Pathname &)
Definition: MediaCurl.cc:374
virtual bool getDoesFileExist(const Pathname &filename) const override
Repeatedly calls doGetDoesFileExist() until it successfully returns, fails unexpectedly,...
Definition: MediaCurl.cc:792
@ OPTION_NO_IFMODSINCE
to not add a IFMODSINCE header if target exists
Definition: MediaCurl.h:43
@ OPTION_NO_REPORT_START
do not send a start ProgressReport
Definition: MediaCurl.h:45
virtual void getFile(const OnMediaLocation &file) const override
Call concrete handler to provide file below attach point.
Definition: MediaCurl.cc:721
static void resetExpectedFileSize(void *clientp, const ByteCount &expectedFileSize)
MediaMultiCurl needs to reset the expected filesize in case a metalink file is downloaded otherwise t...
Definition: MediaCurl.cc:1394
std::string _currentCookieFile
Definition: MediaCurl.h:160
static int aliveCallback(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow)
Callback sending just an alive trigger to the UI, without stats (e.g.
Definition: MediaCurl.cc:1337
static Pathname _cookieFile
Definition: MediaCurl.h:161
static int progressCallback(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow)
Callback reporting download progress.
Definition: MediaCurl.cc:1351
virtual void getFileCopy(const OnMediaLocation &srcFile, const Pathname &targetFilename) const override
Definition: MediaCurl.cc:730
virtual void doGetFileCopy(const OnMediaLocation &srcFile, const Pathname &targetFilename, callback::SendReport< DownloadProgressReport > &_report, RequestOptions options=OPTION_NONE) const
Definition: MediaCurl.cc:1078
std::string _lastRedirect
to log/report redirections
Definition: MediaCurl.h:163
Url clearQueryString(const Url &url) const
Definition: MediaCurl.cc:369
virtual void attachTo(bool next=false) override
Call concrete handler to attach the media.
Definition: MediaCurl.cc:653
virtual void getDirInfo(std::list< std::string > &retlist, const Pathname &dirname, bool dots=true) const override
Call concrete handler to provide a content list of directory on media via retlist.
Definition: MediaCurl.cc:1321
char _curlError[CURL_ERROR_SIZE]
Definition: MediaCurl.h:167
void doGetFileCopyFile(const OnMediaLocation &srcFile, const Pathname &dest, FILE *file, callback::SendReport< DownloadProgressReport > &report, RequestOptions options=OPTION_NONE) const
Definition: MediaCurl.cc:1188
void checkProtocol(const Url &url) const
check the url is supported by the curl library
Definition: MediaCurl.cc:381
MediaCurl(const Url &url_r, const Pathname &attach_point_hint_r)
Definition: MediaCurl.cc:334
void evaluateCurlCode(const zypp::Pathname &filename, CURLcode code, bool timeout) const
Evaluates a curl return code and throws the right MediaException filename Filename being downloaded c...
Definition: MediaCurl.cc:823
virtual bool checkAttachPoint(const Pathname &apoint) const override
Verify if the specified directory as attach point (root) as requires by the particular media handler ...
Definition: MediaCurl.cc:688
virtual void getDir(const Pathname &dirname, bool recurse_r) const override
Call concrete handler to provide directory content (not recursive!) below attach point.
Definition: MediaCurl.cc:1288
bool authenticate(const std::string &availAuthTypes, bool firstTry) const
Definition: MediaCurl.cc:1404
static CURL * progressCallback_getcurl(void *clientp)
Definition: MediaCurl.cc:1367
virtual void releaseFrom(const std::string &ejectDev) override
Call concrete handler to release the media.
Definition: MediaCurl.cc:714
virtual void disconnectFrom() override
Definition: MediaCurl.cc:695
virtual bool doGetDoesFileExist(const Pathname &filename) const
Definition: MediaCurl.cc:971
curl_slist * _customHeaders
Definition: MediaCurl.h:168
std::string getAuthHint() const
Return a comma separated list of available authentication methods supported by server.
Definition: MediaCurl.cc:1375
Just inherits Exception to separate media exceptions.
bool isUseableAttachPoint(const Pathname &path, bool mtab=true) const
Ask media manager, if the specified path is already used as attach point or if there are another atta...
Url url() const
Url used.
Definition: MediaHandler.h:503
virtual bool checkAttachPoint(const Pathname &apoint) const
Verify if the specified directory as attach point (root) as requires by the particular media handler ...
void disconnect()
Use concrete handler to isconnect media.
Pathname createAttachPoint() const
Try to create a default / temporary attach point.
void setMediaSource(const MediaSourceRef &ref)
Set new media source reference.
Pathname localPath(const Pathname &pathname) const
Files provided will be available at 'localPath(filename)'.
const Url _url
Url to handle.
Definition: MediaHandler.h:113
void getDirectoryYast(std::list< std::string > &retlist, const Pathname &dirname, bool dots=true) const
Retrieve and if available scan dirname/directory.yast.
void setAttachPoint(const Pathname &path, bool temp)
Set a new attach point.
Pathname attachPoint() const
Return the currently used attach point.
Common baseclass for MediaCurl and MediaNetwork.
Url getFileUrl(const Pathname &filename) const
concatenate the attach url and the filename to a complete download url
Media source internally used by MediaManager and MediaHandler.
Definition: MediaSource.h:37
const std::string & hint() const
comma separated list of available authentication types
Holds transfer setting.
const std::string & password() const
auth password
long maxDownloadSpeed() const
Maximum download speed (bytes per second)
long connectTimeout() const
connection timeout
const std::string & authType() const
get the allowed authentication types
long timeout() const
transfer timeout
void setUsername(const std::string &val_r)
sets the auth username
const Pathname & clientCertificatePath() const
SSL client certificate file.
std::string userPassword() const
returns the user and password as a user:pass string
long minDownloadSpeed() const
Minimum download speed (bytes per second) until the connection is dropped.
const Headers & headers() const
returns a list of all added headers (trimmed)
const std::string & proxy() const
proxy host
const Pathname & clientKeyPath() const
SSL client key file.
void setUserAgentString(std::string &&val_r)
sets the user agent ie: "Mozilla v3" (trims)
void addHeader(std::string &&val_r)
add a header, on the form "Foo: Bar" (trims)
std::string proxyUserPassword() const
returns the proxy user and password as a user:pass string
bool verifyHostEnabled() const
Whether to verify host for ssl.
const std::string & userAgentString() const
user agent string (trimmed)
void setPassword(const std::string &val_r)
sets the auth password
bool headRequestsAllowed() const
whether HEAD requests are allowed
bool proxyEnabled() const
proxy is enabled
const std::string & username() const
auth username
const std::string & proxyUsername() const
proxy auth username
void setAuthType(const std::string &val_r)
set the allowed authentication types
const Pathname & certificateAuthoritiesPath() const
SSL certificate authorities path ( default: /etc/ssl/certs )
bool verifyPeerEnabled() const
Whether to verify peer for ssl.
#define EXPLICITLY_NO_PROXY
Definition: curlhelper_p.h:21
void fillSettingsFromUrl(const Url &url, media::TransferSettings &s)
Fills the settings structure using options passed on the url for example ?timeout=x&proxy=foo.
Definition: curlhelper.cc:177
size_t log_redirects_curl(char *ptr, size_t size, size_t nmemb, void *userdata)
Definition: curlhelper.cc:137
const char * anonymousIdHeader()
Definition: MediaCurl.cc:251
void globalInitCurlOnce()
Definition: curlhelper.cc:64
const char * distributionFlavorHeader()
Definition: MediaCurl.cc:265
std::string curlUnEscape(std::string text_r)
Definition: curlhelper.cc:360
void setupZYPP_MEDIA_CURL_DEBUG(CURL *curl)
Setup CURLOPT_VERBOSE and CURLOPT_DEBUGFUNCTION according to env::ZYPP_MEDIA_CURL_DEBUG.
Definition: curlhelper.cc:124
const char * agentString()
Definition: MediaCurl.cc:279
void fillSettingsSystemProxy(const Url &url, media::TransferSettings &s)
Reads the system proxy configuration and fills the settings structure proxy information.
Definition: curlhelper.cc:325
Url clearQueryString(const Url &url)
Definition: curlhelper.cc:367
int ZYPP_MEDIA_CURL_IPRESOLVE()
4/6 to force IPv4/v6
Definition: curlhelper.cc:45
mode_t applyUmaskTo(mode_t mode_r)
Modify mode_r according to the current umask ( mode_r & ~getUmask() ).
Definition: PathInfo.h:789
int unlink(const Pathname &path)
Like 'unlink'.
Definition: PathInfo.cc:700
int assert_file_mode(const Pathname &path, unsigned mode)
Like assert_file but enforce mode even if the file already exists.
Definition: PathInfo.cc:1202
std::list< DirEntry > DirContent
Returned by readdir.
Definition: PathInfo.h:518
shared_ptr< CurlAuthData > CurlAuthData_Ptr
Definition: curlauthdata.h:102
shared_ptr< AuthData > AuthData_Ptr
Definition: authdata.h:79
std::string numstring(char n, int w=0)
Definition: String.h:289
std::string form(const char *format,...) __attribute__((format(printf
Printf style construction of std::string.
Definition: String.cc:36
bool strToBool(const C_Str &str, bool default_r)
Parse str into a bool depending on the default value.
Definition: String.h:429
std::string trim(const std::string &s, const Trim trim_r)
Definition: String.cc:223
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:2
AutoDispose< const Pathname > ManagedFile
A Pathname plus associated cleanup code to be executed when path is no longer needed.
Definition: ManagedFile.h:27
Bottleneck filtering all DownloadProgressReport issued from Media[Muli]Curl.
Definition: MediaCurl.cc:50
void report(const UserData &userData_r=UserData()) override
The most generic way of sending/receiving data.
Definition: MediaCurl.cc:67
Action problem(const Url &file_r, Error error_r, const std::string &description_r) override
Definition: MediaCurl.cc:102
OptionalDownloadProgressReport(bool isOptional=false)
Definition: MediaCurl.cc:53
std::chrono::steady_clock::time_point TimePoint
Definition: MediaCurl.cc:51
void finish(const Url &file_r, Error error_r, const std::string &reason_r) override
Definition: MediaCurl.cc:108
bool progress(int value_r, const Url &file_r, double dbps_avg_r=-1, double dbps_current_r=-1) override
Download progress.
Definition: MediaCurl.cc:83
void start(const Url &file_r, Pathname localfile_r) override
Definition: MediaCurl.cc:71
ByteCount _expectedFileSize
Definition: MediaCurl.cc:167
void updateStats(double dltotal=0.0, double dlnow=0.0)
Definition: MediaCurl.cc:197
int _dnlPercent
Percent completed or 0 if _dnlTotal is unknown.
Definition: MediaCurl.cc:179
time_t _timeRcv
Start of no-data timeout.
Definition: MediaCurl.cc:172
ByteCount expectedFileSize() const
Definition: MediaCurl.cc:155
time_t _timeLast
Start last period(~1sec)
Definition: MediaCurl.cc:171
int reportProgress() const
Definition: MediaCurl.cc:240
double _drateLast
Download rate in last period.
Definition: MediaCurl.cc:182
bool timeoutReached() const
Definition: MediaCurl.cc:149
double _dnlTotal
Bytes to download or 0 if unknown.
Definition: MediaCurl.cc:175
void expectedFileSize(ByteCount newval_r)
Definition: MediaCurl.cc:158
double _dnlNow
Bytes downloaded now.
Definition: MediaCurl.cc:177
bool fileSizeExceeded() const
Definition: MediaCurl.cc:152
double _drateTotal
Download rate so far.
Definition: MediaCurl.cc:181
zypp::callback::SendReport< zypp::media::DownloadProgressReport > * report
Definition: MediaCurl.cc:168
double _dnlLast
Bytes downloaded at period start.
Definition: MediaCurl.cc:176
time_t _timeStart
Start total stats.
Definition: MediaCurl.cc:170
AutoDispose<int> calling ::close
Definition: AutoDispose.h:302
AutoDispose<FILE*> calling ::fclose
Definition: AutoDispose.h:313
static DistributeReport & instance()
Definition: Callback.h:204
void setReceiver(Receiver &rec_r)
Definition: Callback.h:213
virtual void reportend()
Definition: Callback.h:191
virtual void reportbegin()
Definition: Callback.h:189
virtual void report(const UserData &userData_r=UserData())
The most generic way of sending/receiving data.
Definition: Callback.h:155
callback::UserData UserData
Definition: Callback.h:151
Structure holding values of curlrc options.
Definition: curlconfig.h:27
std::string proxyuserpwd
Definition: curlconfig.h:49
static int parseConfig(CurlConfig &config, const std::string &filename="")
Parse a curlrc file and store the result in the config structure.
Definition: curlconfig.cc:24
virtual void finish(const Url &, Error, const std::string &)
virtual bool progress(int, const Url &, double dbps_avg=-1, double dbps_current=-1)
Download progress.
virtual void start(const Url &, Pathname)
virtual Action problem(const Url &, Error, const std::string &)
Convenient building of std::string with boost::format.
Definition: String.h:253
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
Definition: Exception.h:440
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:428
#define _(MSG)
Definition: Gettext.h:37
#define DBG
Definition: Logger.h:95
#define MIL
Definition: Logger.h:96
#define ERR
Definition: Logger.h:98
#define WAR
Definition: Logger.h:97