1<?php
2/***********************************************************
3 Copyright (C) 2008-2014 Hewlett-Packard Development Company, L.P.
4 Copyright (C) 2015 Siemens AG
5
6 This program is free software; you can redistribute it and/or
7 modify it under the terms of the GNU General Public License
8 version 2 as published by the Free Software Foundation.
9
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 GNU General Public License for more details.
14
15 You should have received a copy of the GNU General Public License along
16 with this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
18***********************************************************/
19/**
20 * \file cp2foss.php
21 * \brief cp2foss
22 * Import a file (or director or url) into FOSSology for processing.
23 * \exit 0 for success, 1 for failure.
24 */
25
26/**
27 * include common-cli.php directly, common.php can not include common-cli.php
28 * becuase common.php is included before UI_CLI is set
29 */
30require_once("$MODDIR/lib/php/common-cli.php");
31cli_Init();
32
33global $Plugins;
34error_reporting(E_NOTICE & E_STRICT);
35
36$Usage = "Usage: " . basename($argv[0]) . " [options] [archives]
37  Options:
38    -h       = this help message
39    -v       = enable verbose debugging
40    --username string = user name
41    --groupname string = group name
42    --password string = password
43    -c string = Specify the directory for the system configuration
44    -P number = set the permission to public on this upload or not. 1: yes; 0: no
45    -s        = Run synchronously. Don't return until archive already in FOSSology repository.
46                If the archive is a file (see below), then the file can be safely removed.
47
48  Upload from version control system options(you have to specify which type of vcs you are using):
49    -S       = upload from subversion repo
50    -G       = upload from git repo
51    --user string = user name
52    --pass string = password
53
54  FOSSology storage options:
55    -f path  = folder path for placing files (e.g., -f 'Fedora/ISOs/Disk 1')
56               You do not need to specify your top level folder.
57               All paths are under your top level folder.
58    -A       = alphabet folders; organize uploads into folder a-c, d-f, etc.
59    -AA num  = specify the number of letters per folder (default: 3); implies -A
60    -n name  = (optional) name for the upload (default: name it after the file)
61    -d desc  = (optional) description for the update
62
63  FOSSology processing queue options:
64    -Q       = list all available processing agents
65    -q       = specify a comma-separated list of agents, or 'all'
66    NOTE: By default, no analysis agents are queued up.
67    -T       = TEST. No database or repository updates are performed.
68               Test mode enables verbose mode.
69    -I       = ignore scm data scanning
70
71  FOSSology source options:
72    archive  = file, directory, or URL to the archive.
73             If the archive is a URL, then it is retrieved and added.
74             If the archive is a file, then it is used as the source to add.
75             If the archive is a directory, then ALL files under it are
76             recursively added.
77             The archive support globbing - '*', all the matched files will be added.
78                 Note: have to put it in single/double quotes, e.g. '*.php'
79    -        = a single hyphen means the archive list will come from stdin.
80    -X path  = item to exclude when archive is a directory
81             You can specify more than one -X.  For example, to exclude
82             all svn and cvs directories, include the following before the
83             archive's directory path:
84               -X .svn -X .cvs
85    NOTES:
86      If you use -n, then -n must be set BEFORE each archive.
87      If you specify a directory, then -n and -d are ignored.
88      Multiple archives can be specified after each storage option.
89
90  One example, to load a file into one path:
91  cp2foss \\
92    --username USER --password PASSWORD \\
93    -f path -d 'the file' /tmp/file
94  One example, to upload all the php files in /tmp:
95  cp2foss --username USER --password PASSWORD -f path -d 'the file' '/tmp/*.php'
96
97  Deprecated options:
98    -a archive = (deprecated) see archive
99    -e addr    = (deprecated and ignored)
100    -p path    = (deprecated) see -f
101    -R         = (deprecated and ignored)
102    -w         = (deprecated and ignored)
103    -W         = (deprecated and ignored)
104";
105/* Load command-line options */
106global $PG_CONN;
107$Verbose = 0;
108$Test = 0;
109$fossjobs_command = "";
110/************************************************************************/
111/************************************************************************/
112/************************************************************************/
113
114/**
115 * \brief Given an upload name and the number
116 * of letters per bucket, return the bucket folder name.
117 */
118function GetBucketFolder($UploadName, $BucketGroupSize)
119{
120  $Letters = "abcdefghijklmnopqrstuvwxyz";
121  $Numbers = "0123456789";
122  if (empty($UploadName)) {
123    return;
124  }
125  $Name = strtolower(substr($UploadName, 0, 1));
126  /* See if I can find the bucket */
127  if (empty($BucketGroupSize) || ($BucketGroupSize < 1)) {
128    $BucketGroupSize = 3;
129  }
130  for ($i = 0;$i < 26;$i+= $BucketGroupSize) {
131    $Range = substr($Letters, $i, $BucketGroupSize);
132    $Find = strpos($Range, $Name);
133    if ($Find !== false) {
134      if (($BucketGroupSize <= 1) || (strlen($Range) <= 1)) {
135        return ($Range);
136      }
137      return (substr($Range, 0, 1) . '-' . substr($Range, -1, 1));
138    }
139  }
140  /* Not a letter.  Check for numbers */
141  $Find = strpos($Numbers, $Name);
142  if ($Find !== false) {
143    return ("0-9");
144  }
145  /* Not a letter. */
146  return ("Other");
147} /* GetBucketFolder() */
148
149/**
150 * \brief Given a folder path, return the folder_pk.
151 *
152 * \param $FolderPath - path from -f
153 * \param $Parent - parent folder of $FolderPath
154 *
155 * \return folder_pk, 1: 'Software Repository', others: specified folder
156
157 * \note If any part of the folder path does not exist, thenscp cp2foss will create it.
158 * This is recursive!
159 */
160function GetFolder($FolderPath, $Parent = null)
161{
162  $dbManager = $GLOBALS['container']->get('db.manager');
163  global $Verbose;
164  global $Test;
165  if (empty($Parent)) {
166    $Parent = FolderGetTop();
167  }
168  /*/ indicates it's the root folder. Empty folder path ends recursion. */
169  if ($FolderPath == '/') {
170    return ($Parent);
171  }
172  if (empty($FolderPath)) {
173    return ($Parent);
174  }
175  list($folderHead, $folderTail) = explode('/', $FolderPath, 2);
176  if (empty($folderHead)) {
177    return (GetFolder($folderTail, $Parent));
178  }
179  /* See if it exists */
180  $SQL = "SELECT folder_pk FROM folder
181  INNER JOIN foldercontents ON child_id = folder_pk
182  AND foldercontents_mode = '1'
183  WHERE foldercontents.parent_fk = $1 AND folder_name = $2";
184  if ($Verbose) {
185    print "SQL=\n$SQL\n$1=$Parent\n$2=$folderHead\n";
186  }
187
188  $row = $dbManager->getSingleRow($SQL, array($Parent, $folderHead), __METHOD__.".GetFolder.exists");
189  if (empty($row)) {
190    /* Need to create folder */
191    global $Plugins;
192    $P = & $Plugins[plugin_find_id("folder_create") ];
193    if (empty($P)) {
194      print "FATAL: Unable to find folder_create plugin.\n";
195      exit(1);
196    }
197    if ($Verbose) {
198      print "Folder not found: Creating $folderHead\n";
199    }
200    if (!$Test) {
201      $P->create($Parent, $folderHead, "");
202      $row = $dbManager->getSingleRow($SQL, array($Parent, $folderHead), __METHOD__.".GetFolder.exists");
203    }
204  }
205  $Parent = $row['folder_pk'];
206  return (GetFolder($folderTail, $Parent));
207} /* GetFolder() */
208
209/**
210 * \brief Given one object (file or URL), upload it.
211 *
212 * \param $FolderPath - folder path
213 * \param $UploadArchive - upload file(absolute path) or url
214 * \param $UploadName - uploaded file/dir name
215 * \param $UploadDescription - upload description
216 *
217 * \return 1: error, 0: success
218 */
219function UploadOne($FolderPath, $UploadArchive, $UploadName, $UploadDescription, $TarSource = null)
220{
221  $dbManager = $GLOBALS['container']->get('db.manager');
222  global $Verbose;
223  global $Test;
224  global $QueueList;
225  global $fossjobs_command;
226  global $public_flag;
227  global $SysConf;
228  global $VCS;
229  global $vcsuser;
230  global $vcspass;
231  global $TarExcludeList;
232  global $scmarg;
233  $jobqueuepk = 0;
234
235  if (empty($UploadName)) {
236    $text = "UploadName is empty\n";
237    echo $text;
238    return 1;
239  }
240
241  $user_pk = $SysConf['auth']['UserId'];
242  $group_pk = $SysConf['auth']['GroupId'];
243  /* Get the user record and check the PLUGIN_DB_ level to make sure they have at least write access */
244  $UsersRow = GetSingleRec("users", "where user_pk=$user_pk");
245  if ($UsersRow["user_perm"] < PLUGIN_DB_WRITE) {
246    print "You have no permission to upload files into FOSSology\n";
247    return 1;
248  }
249
250  /* Get the folder's primary key */
251  $root_folder_fk = $UsersRow["root_folder_fk"];
252  global $OptionA; /* Should it use bucket names? */
253  if ($OptionA) {
254    global $bucket_size;
255    $FolderPath.= "/" . GetBucketFolder($UploadName, $bucket_size);
256  }
257  $FolderPk = GetFolder($FolderPath, $root_folder_fk);
258  if ($FolderPk == 1) {
259    print "  Uploading to folder: 'Software Repository'\n";
260  } else {
261    print "  Uploading to folder: '$FolderPath'\n";
262  }
263  print "  Uploading as '$UploadName'\n";
264  if (!empty($UploadDescription)) {
265    print "  Upload description: '$UploadDescription'\n";
266  }
267
268  $Mode = (1 << 3); // code for "it came from web upload"
269
270  /* Create the upload for the file */
271  if ($Verbose) {
272    print "JobAddUpload($user_pk, $group_pk, $UploadName,$UploadArchive,$UploadDescription,$Mode,$FolderPk, $public_flag);\n";
273  }
274  if (!$Test) {
275    $Src = $UploadArchive;
276    if (!empty($TarSource)) {
277      $Src = $TarSource;
278    }
279    $UploadPk = JobAddUpload($user_pk, $group_pk, $UploadName, $Src, $UploadDescription, $Mode, $FolderPk, $public_flag);
280    print "  UploadPk is: '$UploadPk'\n";
281    print "  FolderPk is: '$FolderPk'\n";
282  }
283
284  /* Prepare the job: job "wget" */
285  if ($Verbose) {
286    print "JobAddJob($user_pk, $group_pk, wget, $UploadPk);\n";
287  }
288  if (!$Test) {
289    $jobpk = JobAddJob($user_pk, $group_pk, "wget", $UploadPk);
290    if (empty($jobpk) || ($jobpk < 0)) {
291      $text = _("Failed to insert job record");
292      echo $text;
293      return 1;
294    }
295  }
296
297  $jq_args = "$UploadPk - $Src";
298  if ($TarExcludeList) {
299    $jq_args .= " " . $TarExcludeList;
300  }
301  if ($VCS) {
302    $jq_args .= " " . $VCS; // add flags when upload from version control system
303  }
304  if ($vcsuser && $vcspass) {
305    $jq_args .= " --username $vcsuser --password $vcspass ";
306  }
307  if ($Verbose) {
308    print "JobQueueAdd($jobpk, wget_agent, $jq_args, no, NULL);\n";
309  }
310  if (!$Test) {
311    $jobqueuepk = JobQueueAdd($jobpk, "wget_agent", $jq_args, "no", NULL);
312    if (empty($jobqueuepk)) {
313      $text = _("Failed to insert task 'wget' into job queue");
314      echo $text;
315      return 1;
316    }
317  }
318  /* schedule agents */
319  global $Plugins;
320  if ($Verbose) {
321    print "AgentAdd wget_agent and dj2nest.\n";
322  }
323  if (!$Test) {
324    $unpackplugin = &$Plugins[plugin_find_id("agent_unpack") ];
325    $ununpack_jq_pk = $unpackplugin->AgentAdd($jobpk, $UploadPk, $ErrorMsg, array("wget_agent"), $scmarg);
326    if ($ununpack_jq_pk < 0) {
327      echo  $ErrorMsg;
328      return 1;
329    }
330
331    $adj2nestplugin = &$Plugins[plugin_find_id("agent_adj2nest") ];
332    $adj2nest_jq_pk = $adj2nestplugin->AgentAdd($jobpk, $UploadPk, $ErrorMsg, array());
333    if ($adj2nest_jq_pk < 0) {
334      echo $ErrorMsg;
335      return 1;
336    }
337  }
338  if (!empty($QueueList)) {
339    switch ($QueueList) {
340      case 'ALL':
341      case 'all':
342        $Cmd = "$fossjobs_command -U '$UploadPk'";
343        break;
344      default:
345        $Cmd = "$fossjobs_command -U '$UploadPk' -A '$QueueList'";
346        break;
347    }
348    if ($Verbose) {
349      print "CMD=$Cmd\n";
350    }
351    if (!$Test) {
352      system($Cmd);
353    }
354  } else {
355    /* No other agents other than unpack scheduled, attach to unpack*/
356  }
357  global $OptionS; /* Should it run synchronously? */
358  if ($OptionS) {
359    $working = true;
360    $waitCount = 0;
361    while ($working && ($waitCount++ < 30)) {
362      sleep(3);
363      $SQL = "select 1 from jobqueue inner join job on job.job_pk = jobqueue.jq_job_fk where job_upload_fk = $1 and jq_end_bits = 0 and jq_type = 'wget_agent'";
364
365      $row = $dbManager->getSingleRow($SQL, array($UploadPk), __METHOD__.".UploadOne");
366      if (empty($row)) {
367        $working = false;
368      }
369
370    }
371    if ($working) {
372      echo "Gave up waiting for copy completion. Is the scheduler running?";
373      return 1;
374    }
375  }
376} /* UploadOne() */
377
378
379/************************************************************************/
380/************************************************************************/
381/************************************************************************/
382/* Process each parameter */
383$FolderPath = "/";
384$UploadDescription = "";
385$UploadName = "";
386$QueueList = "";
387$TarExcludeList = "";
388$bucket_size = 3;
389$public_flag = 0;
390$scmarg = NULL;
391$OptionS = "";
392
393$user = $passwd = "";
394$group = "";
395$vcsuser = $vcspass= "";
396
397for ($i = 1; $i < $argc; $i ++) {
398  switch ($argv[$i]) {
399    case '-c':
400      $i++;
401      break; /* handled in fo_wrapper */
402    case '-h':
403    case '-?':
404      print $Usage . "\n";
405      exit(0);
406    case '--username':
407      $i++;
408      $user = $argv[$i];
409      break;
410    case '--groupname':
411      $i++;
412      $group = $argv[$i];
413      break;
414    case '--password':
415      $i++;
416      $passwd = $argv[$i];
417      break;
418    case '--user':
419      $i++;
420      $vcsuser = $argv[$i];
421      break;
422    case '--pass':
423      $i++;
424      $vcspass = $argv[$i];
425      break;
426    case '-A': /* use alphabet buckets */
427      $OptionA = true;
428      break;
429    case '-AA': /* use alphabet buckets */
430      $OptionA = true;
431      $i++;
432      $bucket_size = intval($argv[$i]);
433      if ($bucket_size < 1) {
434        $bucket_size = 1;
435      }
436      break;
437    case '-f': /* folder path */
438    case '-p': /* deprecated 'path' to folder */
439      $i++;
440      $FolderPath = $argv[$i];
441      /* idiot check for absolute paths */
442      //print "  Before Idiot Checks: '$FolderPath'\n";
443      /* remove starting and ending / */
444      $FolderPath = preg_replace('@^/*@', "", $FolderPath);
445      $FolderPath = preg_replace('@/*$@', "", $FolderPath);
446      /* Note: the pattern below should probably be generalized to remove everything
447       * up to and including the 1st /, This pattern works in what I've
448      * tested: @^.*\/@ ( I had to escape the / so the comment works!)
449      */
450      $FolderPath = preg_replace("@^S.*? Repository@", "", $FolderPath);
451      $FolderPath = preg_replace('@//*@', "/", $FolderPath);
452      $FolderPath = '/' . $FolderPath;
453      //print "  AFTER Idiot Checks: '$FolderPath'\n";
454
455      break;
456    case '-R': /* obsolete: recurse directories */
457      break;
458    case '-W': /* obsolete: webserver */
459      break;
460    case '-w': /* obsolete: URL switch to use wget */
461      break;
462    case '-d': /* specify upload description */
463      $i++;
464      $UploadDescription = $argv[$i];
465      break;
466    case '-n': /* specify upload name */
467      $i++;
468      $UploadName = $argv[$i];
469      break;
470    case '-Q': /** list all available processing agents */
471      $OptionQ = 1;
472      break;
473    case '-q':
474      $i++;
475      $QueueList = $argv[$i];
476      break;
477    case '-s':
478      $OptionS = true;
479      break;
480    case '-T': /* Test mode */
481      $Test = 1;
482      if (!$Verbose) {
483        $Verbose++;
484      }
485      break;
486    case '-v':
487      $Verbose++;
488      break;
489    case '-X':
490      if (!empty($TarExcludeList)) {
491        $TarExcludeList .= " ";
492      }
493      $i++;
494      $TarExcludeList .= "--exclude '" . $argv[$i] . "'";
495      break;
496    case '-a': /* it's an archive! */
497      /* ignore -a since the next name is a file. */
498      break;
499    case '-': /* it's an archive list from stdin! */
500      $stdin_flag = 1;
501      break;
502    case '-P': /* set the permission to public or not */
503      $i++;
504      if (1 == $argv[$i]) {
505        $public_flag = 1;
506      } else {
507        $public_flag = 0;
508      }
509      break;
510    case '-S': /* upload from subversion repo */
511      $VCS = 'SVN';
512      break;
513    case '-G': /* upload from git repo */
514      $VCS = 'Git';
515      break;
516    case '-I': /* ignore scm data when scanning */
517      $scmarg = '-I';
518      break;
519    default:
520      if (substr($argv[$i], 0, 1) == '-') {
521        print "Unknown parameter: '" . $argv[$i] . "'\n";
522        print $Usage . "\n";
523        exit(1);
524      }
525      /* No hyphen means it is a file! */
526      $UploadArchive = $argv[$i];
527  } /* switch */
528} /* for each parameter */
529
530account_check($user, $passwd, $group); // check username/password
531
532/** list all available processing agents */
533if (!$Test && $OptionQ) {
534  $Cmd = "fossjobs --username $user --groupname $group --password $passwd -c $SYSCONFDIR -a";
535  system($Cmd);
536  exit(0);
537}
538
539/** get archive from stdin */
540if ($stdin_flag) {
541  $Fin = fopen("php://stdin", "r");
542  if (!feof($Fin)) {
543    $UploadArchive = trim(fgets($Fin));
544  }
545  fclose($Fin);
546}
547
548/** compose fossjobs command */
549if ($Verbose) {
550  $fossjobs_command = "fossjobs --username $user --groupname $group --password $passwd -c $SYSCONFDIR -v ";
551} else {
552  $fossjobs_command = "fossjobs --username $user --groupname $group --password $passwd -c $SYSCONFDIR  ";
553}
554
555//print "fossjobs_command is:$fossjobs_command\n";
556
557if (!$UploadArchive) {  // upload is empty
558  print "FATAL: No files to upload were specified.\n";
559  exit(1);
560}
561
562/** get real path, and file name */
563$UploadArchiveTmp = "";
564$UploadArchiveTmp = realpath($UploadArchive);
565if (!$UploadArchiveTmp) {
566  // neither a file nor folder from server?
567  if (filter_var($UploadArchive, FILTER_VALIDATE_URL)) {
568  } else if (strchr($UploadArchive, '*')) {
569    $file_number_cmd = "ls $UploadArchive > /dev/null";
570    system($file_number_cmd, $return_val);
571    if ($return_val) {
572      exit(1); // not files matched
573    }
574    if ("/" != $UploadArchive[0]) { // it is a absolute path
575      $UploadArchive = getcwd()."/".$UploadArchive;
576    }
577  } else {
578    print "Note: it seems that what you want to upload '$UploadArchive' does not exist. \n";
579    exit(1);
580  }
581} else {  // is a file or folder from server
582  $UploadArchive = $UploadArchiveTmp;
583}
584
585if (strlen($UploadArchive) > 0 && empty($UploadName)) {
586  $UploadName = basename($UploadArchive);
587}
588
589if ($vcsuser && $vcspass) {
590  print "Warning: usernames and passwords on the command line are visible to system users with a shell account.  To avoid this you can download your source, then upload.\n";
591}
592
593print "Loading '$UploadArchive'\n";
594print "  Calling UploadOne in 'main': '$FolderPath'\n";
595$res = UploadOne($FolderPath, $UploadArchive, $UploadName, $UploadDescription);
596if ($res) {
597  exit(1); // fail to upload
598}
599exit(0);
600