1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
|
#ifndef _DLCON_H
#define _DLCON_H
#include <string>
#include <list>
#include <map>
#include <set>
//#include <netinet/in.h>
//#include <netdb.h>
#include "tcpconnect.h"
#include "lockable.h"
#include "fileitem.h"
#include "acfg.h"
#include "acbuf.h"
namespace acng
{
struct tDlJob;
typedef std::shared_ptr<tDlJob> tDlJobPtr;
typedef std::list<tDlJobPtr> tDljQueue;
/**
* dlcon is a basic connection broker for download processes.
* It's defacto a slave of the conn class, the active thread is spawned by conn when needed
* and it's finished by its destructor. However, the life time is prolonged if the usage count
* is not down to zero, i.e. when there are more users registered as reader for the file
* downloaded by the agent here then it will continue downloading and block the conn dtor
* until that download is finished or the other client detaches. If a download is active and parent
* conn object calls Stop... then the download will be aborted ASAP.
*
* Internally, a queue of download job items is maintained. Each contains a reference either to
* a full target URL or to a tupple of a list of mirror descriptions (url prefix) and additional
* path suffix for the required file.
*
* In addition, there is a local blacklist which is applied to all download jobs in the queue,
* i.e. remotes marked as faulty there are no longer considered by the subsequent download jobs.
*/
class dlcon : public base_with_mutex
{
public:
dlcon(bool bManualExecution, mstring *xff=nullptr,
IDlConFactory *pConFactory = &g_tcp_con_factory);
~dlcon();
void WorkLoop();
void SignalStop();
bool AddJob(tFileItemPtr m_pItem, const tHttpUrl *pForcedUrl,
const cfg::tRepoData *pRepoDesc,
cmstring *sPatSuffix, LPCSTR reqHead);
mstring m_sXForwardedFor;
private:
//not to be copied
dlcon & operator=(const dlcon&);
dlcon(const dlcon&);
friend struct tDlJob;
tDljQueue m_qNewjobs;
IDlConFactory* m_pConFactory;
#ifdef HAVE_LINUX_EVENTFD
int m_wakeventfd = -1;
#define fdWakeRead m_wakeventfd
#define fdWakeWrite m_wakeventfd
#else
int m_wakepipe[2] = {-1, -1};
#define fdWakeRead m_wakepipe[0]
#define fdWakeWrite m_wakepipe[1]
#endif
// flags and local copies for input parsing
/// remember being attached to an fitem
bool m_bStopASAP;
unsigned m_bManualMode;
/// blacklist for permanently failing hosts, with error message
std::map<std::pair<cmstring,cmstring>, mstring> m_blacklist;
tSS m_sendBuf, m_inBuf;
unsigned ExchangeData(mstring &sErrorMsg, tDlStreamHandle &con, tDljQueue &qActive);
// Disable pipelining for the next # requests. Actually used as crude workaround for the
// concept limitation (because of automata over a couple of function) and its
// impact on download performance.
// The use case: stupid web servers that redirect all requests do that step-by-step, i.e.
// they get a bunch of requests but return only the first response and then flush the buffer
// so we process this response and wish to switch to the new target location (dropping
// the current connection because we don't keep it somehow to background, this is the only
// download agent we have). This manner perverts the whole principle and causes permanent
// disconnects/reconnects. In this case, it's beneficial to disable pipelining and send
// our requests one-by-one. This is done for a while (i.e. the valueof(m_nDisablePling)/2 )
// times before the operation mode returns to normal.
int m_nTempPipelineDisable;
// the default behavior or using or not using the proxy. Will be set
// if access proxies shall no longer be used.
bool m_bProxyTot;
// this is a binary factor, meaning how many reads from buffer are OK when
// speed limiting is enabled
unsigned m_nSpeedLimiterRoundUp = (unsigned(1)<<16)-1;
unsigned m_nSpeedLimitMaxPerTake = MAX_VAL(unsigned);
unsigned m_nLastDlCount=0;
void wake();
};
#define IS_REDIRECT(st) (st == 301 || st == 302 || st == 307)
}
#endif
|
