1
2 /* Web Polygraph http://www.web-polygraph.org/
3 * Copyright 2003-2011 The Measurement Factory
4 * Licensed under the Apache License, Version 2.0 */
5
6 #include "base/polygraph.h"
7
8 #include "base/polyLogCats.h"
9 #include "base/StatIntvlRec.h"
10 #include "runtime/LogComment.h"
11 #include "runtime/ErrorMgr.h"
12 #include "runtime/SharedOpts.h"
13 #include "client/Client.h"
14 #include "client/CltConnMgr.h"
15 #include "client/FtpCxm.h"
16 #include "client/FtpCltXact.h"
17 #include "client/ServerRep.h"
18 #include "csm/oid2Url.h"
19 #include "csm/BodyIter.h"
20 #include "csm/ContentCfg.h"
21 #include "runtime/globals.h"
22 #include "runtime/polyErrors.h"
23 #include "runtime/FtpText.h"
24
FtpCltXact()25 FtpCltXact::FtpCltXact() {
26 FtpCltXact::reset();
27 }
28
reset()29 void FtpCltXact::reset() {
30 CltXact::reset();
31 protoStat = &StatIntvlRec::theFtpStat;
32 theReqOid.type(TheBodilessContentId);
33 theRep.reset();
34 theReqCmd = FtpReq::frcUnknown;
35 theActiveDataWaitingCmd = FtpReq::frcUnknown;
36 theDataConn.reset();
37 theDataConn.logCat(lgcCltSide);
38 theDataConn.protoStat = protoStat;
39 stopDataListen();
40 stopDataChannel();
41 theDataChannelState = dcsNone;
42 //Comment << here << "reset to new data state: " << theDataChannelState << endc;
43 }
44
exec(Connection * const aConn)45 void FtpCltXact::exec(Connection *const aConn) {
46 CltXact::exec(aConn);
47 newState(stHdrWaiting);
48
49 Assert(!theMgr);
50 theMgr = FtpCxm::Get();
51 theMgr->control(this);
52 }
53
finish(Error err)54 void FtpCltXact::finish(Error err) {
55 //Comment << here << "finishing " << err << endc;
56 if (theDataChannelState == dcsNone && !err) // we have seen neither RETR nor error
57 err = errFtpNoDataXfer;
58 stopDataListen();
59 stopDataChannel();
60 stopCtrlChannel(err);
61 CltXact::finish(err);
62 }
63
writeFirst() const64 bool FtpCltXact::writeFirst() const {
65 return false;
66 }
67
wantsToWrite() const68 bool FtpCltXact::wantsToWrite() const {
69 // we usually want to write after reading a reply, except
70 // when we need to wait for the data connection to succeed
71 return theDataChannelState != dcsConnecting;
72 }
73
getPipeline()74 PipelinedCxm *FtpCltXact::getPipeline() {
75 return 0; // FTP cannot be a pipelined transaction
76 }
77
pipeline(PipelinedCxm *)78 void FtpCltXact::pipeline(PipelinedCxm *) {
79 // FTP does not support pipelining so we will not change theMgr
80 }
81
proxyStatAuth() const82 AuthPhaseStat::Scheme FtpCltXact::proxyStatAuth() const {
83 return theOid.authCred() ? AuthPhaseStat::sFtp : AuthPhaseStat::sNone;
84 }
85
stopCtrlChannel(const Error & err)86 void FtpCltXact::stopCtrlChannel(const Error &err) {
87 if (theMgr) {
88 if (err)
89 theMgr->noteAbort(this);
90 else
91 theMgr->noteDone(this);
92 theMgr->release(this);
93 theMgr = 0;
94 }
95 }
96
controlledPostRead(bool & needMore)97 bool FtpCltXact::controlledPostRead(bool &needMore) {
98 Assert(theState == stHdrWaiting);
99
100 if (Error err = handleReplies()) {
101 finish(err);
102 return false;
103 }
104
105 if (finishIfDone() || !theMgr)
106 return false;
107
108 needMore = theState == stHdrWaiting;
109 return true;
110 }
111
finishIfDone()112 bool FtpCltXact::finishIfDone() {
113 //Comment << here << "ctrl: " << theState << " data: " << theDataChannelState << endc;
114 //Comment << here << (theState == stDone) << " / " << (theDataChannelState == dcsDone) << " || " << (theDataChannelState == dcsNone) << endc;
115 // do not quit if we are not done with the control channel
116 if (theState != stDone)
117 return false;
118
119 // quit if we are done with the data channel
120 if (theDataChannelState == dcsDone || theDataChannelState == dcsNone) {
121 finish(0);
122 return true;
123 }
124
125 return false;
126 }
127
handleReplies()128 Error FtpCltXact::handleReplies() {
129 IOBuf &buf = theConn->theRdBuf;
130 //Comment << "FTP reply (" << buf.contSize() << "): " << endc; printMsg(buf.content(), buf.contSize());
131
132 // due to Preliminary and Intermediate replies, we might read several
133 // server replies at once; handle them one by one until we are done or err
134 while (theRep.parse(buf.content(), buf.contSize())) {
135 //Comment << "FTP reply size: " << theRep.size() << endc;
136 //Comment << here << "ctrl: " << theState << " data: " << theDataChannelState << endc;
137 theRepSize.expect(theRep.size()); // XXX: ignores data channel
138 consume(theRep.size()); // XXX: does not work for data/ctrl conn split
139 if (const Error err = interpretReply())
140 return err;
141
142 //Comment << here << "ctrl: " << theState << " data: " << theDataChannelState << endc;
143
144 if (theRep.code() >= 200) {
145 if (!buf.empty()) { // leftovers?
146 if (ReportError(errExtraRepData)) { // XXX: add and use errExtraReply
147 Comment << "FTP reply leftovers (" << buf.contSize() <<
148 ")" << endc;
149 if (TheOpts.theDumpFlags(dumpErr, dumpAny))
150 printMsg(buf.content(), buf.contSize());
151 }
152 return errOther;
153 }
154
155 theReqSize.reset(); // XXX: why? should we do it after 150?
156 if (theState == stHdrWaiting)
157 newState(stSpaceWaiting); // signal that no more data is needed
158 return 0;
159 }
160
161 theRep.reset();
162 theRepSize.reset();
163 // if we got a Positive Preliminary reply, check if is there more
164 }
165
166 if (buf.full()) // command too big
167 return errFtpHugeCmd;
168
169 if (theConn->atEof()) { // premature end of reply stream
170 if (buf.empty())
171 return errFtpPrematureEndOfCtrl; // expected a reply, got nothing
172 else
173 return errFtpPrematureEndOfMsg; // we did not get the entire reply
174 }
175
176 Assert(theState == stHdrWaiting);
177 return 0; // need more data to parse the header
178 }
179
makeReq(WrBuf & buf)180 void FtpCltXact::makeReq(WrBuf &buf) {
181 Assert(!theReqSize.expected().known());
182 ofixedstream os(buf.space(), buf.spaceSize());
183 switch (theReqCmd) {
184 case FtpReq::frcUSER:
185 makeUser(os);
186 break;
187 case FtpReq::frcPASS:
188 makePass(os);
189 break;
190 case FtpReq::frcTYPE:
191 makeType(os);
192 break;
193 case FtpReq::frcPASV:
194 makePasv(os);
195 break;
196 case FtpReq::frcPORT:
197 makePort(os);
198 break;
199 case FtpReq::frcRETR:
200 makeRetr(os);
201 break;
202 case FtpReq::frcSTOR:
203 makeStor(os);
204 break;
205 case FtpReq::frcQUIT:
206 makeQuit(os);
207 break;
208 default:
209 // if we get here, it is an intermal error, but we cannot finish()
210 // TODO: need exceptions to properly handle this and other errors
211 ReportError(errFtpCmdSequence);
212 makeQuit(os);
213 }
214 theReqSize.expect(Size(os.tellp()));
215 buf.appended(theReqSize.expected());
216 //Comment << "FTP request (" << buf.contSize() << "): " << endc; printMsg(buf.content(), buf.contSize());
217 }
218
interpretReply()219 Error FtpCltXact::interpretReply() {
220 Assert(theState == stHdrWaiting);
221 if (theRep.code() < 100 || theRep.code() >= 400)
222 return errFtpCommandFailed;
223
224 // Interpret the server response and make sure we are making progress
225 // according to client-side plan
226
227 Error err = 0;
228 // XXX: use named (enum) constants below
229 // TODO: use a const transition table: rep.code -> expected state,
230 // next state?
231 switch (theRep.code()) {
232 case 220: // server greating
233 err = transition(FtpReq::frcUnknown, FtpReq::frcUSER);
234 if (!err && theSrvRep) {
235 // we note request here to mimic HTTP; TODO: redesign?
236 theSrvRep->noteRequest();
237 if (true) // XXX: but there are FTP proxies; detect polysrv
238 theSrvRep->noteFirstHandResponse();
239 }
240 break;
241 case 331: // username accepted, password required
242 err = transition(FtpReq::frcUSER, FtpReq::frcPASS);
243 break;
244 case 230: // password accepted.
245 err = transition(FtpReq::frcPASS, FtpReq::frcTYPE);
246 break;
247 case 200:
248 if (theReqCmd == FtpReq::frcTYPE) { // entering binary mode
249 if (theOwner->usesPassiveFtp()) {
250 err = transition(FtpReq::frcTYPE, FtpReq::frcPASV);
251 theOid.passive(true);
252 } else {
253 err = transition(FtpReq::frcTYPE, FtpReq::frcPORT);
254 theOid.active(true);
255 if (!err)
256 err = startDataListen();
257 }
258 } else // PORT accepted
259 err = transition(FtpReq::frcPORT, theOid.put() ? FtpReq::frcSTOR : FtpReq::frcRETR);
260 break;
261 case 227: // passive address
262 err = transition(FtpReq::frcPASV, theOid.put() ? FtpReq::frcSTOR : FtpReq::frcRETR);
263 if (!err)
264 err = interpretPasv();
265 break;
266 case 125: // Data connection already open; transfer starting.
267 case 150: // Opening data connection.
268 err = checkDataChannel();
269 if (!err && theReqCmd != FtpReq::frcUnknown)
270 err = transition(theReqCmd, FtpReq::frcUnknown);
271 break;
272 case 226: // Operation completed
273 err = checkDataChannel(); // in case an optional(?) 125 or 150 was not received
274 if (!err)
275 err = transition(theReqCmd, FtpReq::frcQUIT);
276 break;
277 case 221: // Bye.
278 err = transition(FtpReq::frcQUIT, FtpReq::frcUnknown);
279 theConn->lastUse(true); // no pconn support yet
280 stopCtrlChannel(err);
281 if (!err)
282 newState(stDone); // XXX: misplaced, not the last call
283 break;
284 default:
285 Assert(false); // XXX: report unknown status codes
286 }
287
288 return err;
289 }
290
transition(FtpReq::Command from,FtpReq::Command to)291 Error FtpCltXact::transition(FtpReq::Command from, FtpReq::Command to) {
292 //Comment << here << "transition: " << theReqCmd << " =? " << from << " -> " << to << endc;
293 if (theReqCmd != from)
294 return errFtpCommandFailed; // XXX: report details and use errOther
295
296 theReqCmd = to;
297 return 0;
298 }
299
300 // do not call Client::originAuthScheme: assume FTP always requires authentication
makeUser(ostream & os)301 void FtpCltXact::makeUser(ostream &os) {
302 os << ftpReqUserPfx;
303 if (genCredentials()) { // TODO: call in start()
304 os << theCred.name();
305 Should(theOid.authCred()); // Client sets this
306 } else {
307 os << ftpReqUserAnonym;
308 theOid.authCred(true); // anonymous but still authenticated
309 }
310 os << ftpReqUserSfx;
311 theReqCmd = FtpReq::frcUSER;
312 }
313
makePass(ostream & os)314 void FtpCltXact::makePass(ostream &os) {
315 os << ftpReqPassPfx;
316 if (theCred.image())
317 os << theCred.password();
318 else
319 os << ftpReqPassAnonym;
320 os << ftpReqPassSfx;
321 theReqCmd = FtpReq::frcPASS;
322 }
323
makeType(ostream & os)324 void FtpCltXact::makeType(ostream &os) {
325 os << ftpReqTypeBinary;
326 theReqCmd = FtpReq::frcTYPE;
327 }
328
makePasv(ostream & os)329 void FtpCltXact::makePasv(ostream &os) {
330 os << ftpReqPasv;
331 theReqCmd = FtpReq::frcPASV;
332 }
333
makePort(ostream & os)334 void FtpCltXact::makePort(ostream &os) {
335 Assert(theDataPort >= 0);
336 const NetAddr addr(theOwner->host().addrN(), theDataPort);
337 FtpMsg::PrintAddr(os << ftpReqPort, addr) << crlf;
338 theDataPort = -1;
339 theReqCmd = FtpReq::frcPORT;
340 }
341
makeRetr(ostream & os)342 void FtpCltXact::makeRetr(ostream &os) {
343 Assert(theDataChannelState == dcsConnected || theDataChannelState == dcsListening);
344 os << ftpReqRetr;
345 Oid2UrlPath(theOid, os) << crlf;
346 theReqCmd = FtpReq::frcRETR;
347 if (theOid.foreignUrl())
348 TheEmbedStats.foreignUrlRequested++;
349 }
350
makeStor(ostream & os)351 void FtpCltXact::makeStor(ostream &os) {
352 Assert(theDataChannelState == dcsConnected || theDataChannelState == dcsListening);
353 Assert(!theReqContentCfg);
354 Assert(!theBodyIter);
355
356 os << ftpReqStor;
357 Oid2UrlPath(theOid, os) << crlf;
358 theReqCmd = FtpReq::frcSTOR;
359 if (theOid.foreignUrl())
360 TheEmbedStats.foreignUrlRequested++;
361
362 theReqContentCfg = theOwner->selectReqContent(theOid, theReqOid);
363 theBodyIter = theReqContentCfg->getBodyIter(theReqOid);
364 theBodyIter->start(&theDataConn.theWrBuf);
365 }
366
makeQuit(ostream & os)367 void FtpCltXact::makeQuit(ostream &os) {
368 os << ftpReqQuit;
369 theReqCmd = FtpReq::frcQUIT;
370 }
371
interpretPasv()372 Error FtpCltXact::interpretPasv() {
373 const NetAddr addr(FtpMsg::ParseAddr(theRep.data()));
374 return addr ? startDataChannel(addr) : errFtpBadPasv;
375 }
376
checkDataChannel()377 Error FtpCltXact::checkDataChannel() {
378 Error err = 0;
379 if (dataCommand(theReqCmd)) {
380 if (theDataChannelState == dcsConnected)
381 err = kickDataChannel();
382 else if (theDataChannelState == dcsListening)
383 theActiveDataWaitingCmd = theReqCmd;
384 }
385 return err;
386 }
387
kickDataChannel()388 Error FtpCltXact::kickDataChannel() {
389 if (theDataChannelState != dcsConnected)
390 return errFtpCommandFailed;
391
392 theDataChannelState = dcsInProgress;
393 //Comment << here << "starting to read data; new data state: " << theDataChannelState << endc;
394 const FtpReq::Command reqCmd = dataCommand(theReqCmd) ?
395 theReqCmd : theActiveDataWaitingCmd;
396 if (reqCmd == FtpReq::frcRETR)
397 theDataConn.theRd.start(this);
398 else if (ShouldUs(reqCmd == FtpReq::frcSTOR))
399 theDataConn.theWr.start(this);
400
401 theActiveDataWaitingCmd = FtpReq::frcUnknown;
402 return 0;
403 }
404
startDataChannel(const NetAddr & addr)405 Error FtpCltXact::startDataChannel(const NetAddr &addr) {
406 if (theDataChannelState != dcsNone)
407 return errFtpCmdSequence;
408 // ConnMgr is used to get credentials for SOCKS proxy
409 theDataConn.mgr(theOwner->connMgr());
410 theDataConn.startUse();
411 Should(theDataConn.connect(addr, SockOpt(), theOwner->portMgr(), theOwner->socksProxy()));
412 theDataConn.theWr.start(this);
413 theDataChannelState = dcsConnecting;
414 //Comment << here << "new data state: " << theDataChannelState << endc;
415 return 0;
416 }
417
startDataListen()418 Error FtpCltXact::startDataListen() {
419 if (theDataChannelState != dcsNone)
420 return errFtpCmdSequence;
421
422 Assert(theDataPort < 0);
423 theSock = theOwner->makeListenSocket(theOwner->portMgr());
424 if (!theSock) {
425 if (ReportError2(Error::Last(), lgcCltSide))
426 Comment << "fyi: FTP client could not create listening socket at " << theOwner->host() << endc;
427 return errOther;
428 }
429
430 theDataPort = theSock.lport();
431 Assert(theDataPort >= 0);
432
433 theReserv = TheFileScanner->setFD(theSock.fd(), dirRead, this);
434 theDataChannelState = dcsListening;
435
436 return 0;
437 }
438
stopDataChannel()439 void FtpCltXact::stopDataChannel() {
440 theDataConn.closeNow();
441 //Comment << here << "new data state: " << theDataChannelState << endc;
442 }
443
stopDataListen()444 void FtpCltXact::stopDataListen() {
445 if (theReserv)
446 TheFileScanner->clearRes(theReserv);
447 if (theSock)
448 theSock.close();
449 theDataPort = -1;
450 }
451
noteReadReady(int fd)452 void FtpCltXact::noteReadReady(int fd) {
453 if (fd == theSock.fd())
454 acceptDataConnection();
455 else
456 if (fd == theDataConn.fd())
457 noteDataChannelReadReady();
458 else
459 Assert(false);
460 }
461
noteWriteReady(int fd)462 void FtpCltXact::noteWriteReady(int fd) {
463 if (fd == theDataConn.fd())
464 noteDataChannelWriteReady();
465 else
466 Assert(false);
467 }
468
acceptDataConnection()469 void FtpCltXact::acceptDataConnection() {
470 Assert(theDataChannelState == dcsListening);
471 Assert(!theDataConn.sock());
472 theDataConn.startUse();
473 theDataConn.protoStat = protoStat;
474 bool fatal = false;
475 Must(theDataConn.accept(theSock, SockOpt(), fatal));
476 //Comment << theSock.fd() << " accepted " << theDataConn.sock().fd() << endc;
477
478 stopDataListen();
479 theDataChannelState = dcsConnected;
480 if (dataCommand(theReqCmd) || dataCommand(theActiveDataWaitingCmd))
481 kickDataChannel();
482 else
483 finish(errOther);
484 }
485
noteDataChannelReadReady()486 void FtpCltXact::noteDataChannelReadReady() {
487 Assert(theDataChannelState == dcsInProgress);
488
489 // TODO: support simulated I/O aborts, MD5 checks
490 const Size sz = theDataConn.read();
491 //Comment << "FTP data read: " << sz << " eof: " << theDataConn.atEof() << endc;
492 if (theDataConn.bad()) {
493 finish(errOther);
494 return;
495 }
496
497 if (sz > 0) {
498 theRepSize.got(sz);
499 theDataConn.theRdBuf.consumed(sz);
500 return; // only EOF signals the end of data
501 }
502
503 if (theDataConn.atEof()) {
504 stopDataChannel();
505 theDataChannelState = dcsDone;
506 //Comment << here << "new data state: " << theDataChannelState << endc;
507 finishIfDone();
508 return;
509 }
510 }
511
noteDataChannelWriteReady()512 void FtpCltXact::noteDataChannelWriteReady() {
513 //Comment << here << "ctrl: " << theState << " data: " << theDataChannelState << endc;
514 switch (theDataChannelState) {
515 case dcsConnecting:
516 if (const Error err = theDataConn.sock().error()) {
517 //Comment << here << "connect err: " << err << endc;
518 finish(err);
519 return;
520 }
521 if (theDataConn.socksProxy() &&
522 !theDataConn.socksConnected())
523 theDataConn.socksWrite();
524 else {
525 theDataConn.theWr.stop(this);
526 theDataChannelState = dcsConnected;
527 //Comment << here << "new data state: " << theDataChannelState << endc;
528 // we can now send RETR/STOR
529 // the response will kick reading on a data channel
530 theMgr->resumeWriting(this);
531 }
532 break;
533 case dcsInProgress:
534 Assert(theBodyIter);
535 theBodyIter->pour();
536 theDataConn.write();
537 if ((theBodyIter->pouredAll() &&
538 theDataConn.theWrBuf.empty()) ||
539 theDataConn.bad()) {
540 stopDataChannel();
541 theBodyIter->putBack();
542 theBodyIter = 0;
543 theDataChannelState = theDataConn.bad() ?
544 dcsAborted : dcsDone;
545 } else
546 if (!theDataConn.theWr.theReserv)
547 theDataConn.theWr.start(this);
548 break;
549 default:
550 Assert(false);
551 }
552 }
553
controlledFill(bool & needMore)554 bool FtpCltXact::controlledFill(bool &needMore) {
555 // do nothing while data channel is connecting
556 // if (theDataChannelState == dcsConnecting)
557 // return true;
558
559 return CltXact::controlledFill(needMore);
560 }
561
controlledPostWrite(Size & size,bool & needMore)562 bool FtpCltXact::controlledPostWrite(Size &size, bool &needMore) {
563 // do nothing while data channel is connecting
564 // if (theDataChannelState == dcsConnecting)
565 // return true;
566
567 return CltXact::controlledPostWrite(size, needMore);
568 }
569
controlledMasterRead()570 bool FtpCltXact::controlledMasterRead() {
571 // do nothing while data channel is reading
572 // if (theDataChannelState == dcsInProgress)
573 // return true;
574
575 return CltXact::controlledMasterRead();
576 }
577