1<?php 2/*********************************************************** 3 Copyright (C) 2008-2014 Hewlett-Packard Development Company, L.P. 4 Copyright (C) 2015 Siemens AG 5 6 This program is free software; you can redistribute it and/or 7 modify it under the terms of the GNU General Public License 8 version 2 as published by the Free Software Foundation. 9 10 This program is distributed in the hope that it will be useful, 11 but WITHOUT ANY WARRANTY; without even the implied warranty of 12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 GNU General Public License for more details. 14 15 You should have received a copy of the GNU General Public License along 16 with this program; if not, write to the Free Software Foundation, Inc., 17 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18***********************************************************/ 19/** 20 * \file cp2foss.php 21 * \brief cp2foss 22 * Import a file (or director or url) into FOSSology for processing. 23 * \exit 0 for success, 1 for failure. 24 */ 25 26/** 27 * include common-cli.php directly, common.php can not include common-cli.php 28 * becuase common.php is included before UI_CLI is set 29 */ 30require_once("$MODDIR/lib/php/common-cli.php"); 31cli_Init(); 32 33global $Plugins; 34error_reporting(E_NOTICE & E_STRICT); 35 36$Usage = "Usage: " . basename($argv[0]) . " [options] [archives] 37 Options: 38 -h = this help message 39 -v = enable verbose debugging 40 --username string = user name 41 --groupname string = group name 42 --password string = password 43 -c string = Specify the directory for the system configuration 44 -P number = set the permission to public on this upload or not. 1: yes; 0: no 45 -s = Run synchronously. Don't return until archive already in FOSSology repository. 46 If the archive is a file (see below), then the file can be safely removed. 47 48 Upload from version control system options(you have to specify which type of vcs you are using): 49 -S = upload from subversion repo 50 -G = upload from git repo 51 --user string = user name 52 --pass string = password 53 54 FOSSology storage options: 55 -f path = folder path for placing files (e.g., -f 'Fedora/ISOs/Disk 1') 56 You do not need to specify your top level folder. 57 All paths are under your top level folder. 58 -A = alphabet folders; organize uploads into folder a-c, d-f, etc. 59 -AA num = specify the number of letters per folder (default: 3); implies -A 60 -n name = (optional) name for the upload (default: name it after the file) 61 -d desc = (optional) description for the update 62 63 FOSSology processing queue options: 64 -Q = list all available processing agents 65 -q = specify a comma-separated list of agents, or 'all' 66 NOTE: By default, no analysis agents are queued up. 67 -T = TEST. No database or repository updates are performed. 68 Test mode enables verbose mode. 69 -I = ignore scm data scanning 70 71 FOSSology source options: 72 archive = file, directory, or URL to the archive. 73 If the archive is a URL, then it is retrieved and added. 74 If the archive is a file, then it is used as the source to add. 75 If the archive is a directory, then ALL files under it are 76 recursively added. 77 The archive support globbing - '*', all the matched files will be added. 78 Note: have to put it in single/double quotes, e.g. '*.php' 79 - = a single hyphen means the archive list will come from stdin. 80 -X path = item to exclude when archive is a directory 81 You can specify more than one -X. For example, to exclude 82 all svn and cvs directories, include the following before the 83 archive's directory path: 84 -X .svn -X .cvs 85 NOTES: 86 If you use -n, then -n must be set BEFORE each archive. 87 If you specify a directory, then -n and -d are ignored. 88 Multiple archives can be specified after each storage option. 89 90 One example, to load a file into one path: 91 cp2foss \\ 92 --username USER --password PASSWORD \\ 93 -f path -d 'the file' /tmp/file 94 One example, to upload all the php files in /tmp: 95 cp2foss --username USER --password PASSWORD -f path -d 'the file' '/tmp/*.php' 96 97 Deprecated options: 98 -a archive = (deprecated) see archive 99 -e addr = (deprecated and ignored) 100 -p path = (deprecated) see -f 101 -R = (deprecated and ignored) 102 -w = (deprecated and ignored) 103 -W = (deprecated and ignored) 104"; 105/* Load command-line options */ 106global $PG_CONN; 107$Verbose = 0; 108$Test = 0; 109$fossjobs_command = ""; 110/************************************************************************/ 111/************************************************************************/ 112/************************************************************************/ 113 114/** 115 * \brief Given an upload name and the number 116 * of letters per bucket, return the bucket folder name. 117 */ 118function GetBucketFolder($UploadName, $BucketGroupSize) 119{ 120 $Letters = "abcdefghijklmnopqrstuvwxyz"; 121 $Numbers = "0123456789"; 122 if (empty($UploadName)) { 123 return; 124 } 125 $Name = strtolower(substr($UploadName, 0, 1)); 126 /* See if I can find the bucket */ 127 if (empty($BucketGroupSize) || ($BucketGroupSize < 1)) { 128 $BucketGroupSize = 3; 129 } 130 for ($i = 0;$i < 26;$i+= $BucketGroupSize) { 131 $Range = substr($Letters, $i, $BucketGroupSize); 132 $Find = strpos($Range, $Name); 133 if ($Find !== false) { 134 if (($BucketGroupSize <= 1) || (strlen($Range) <= 1)) { 135 return ($Range); 136 } 137 return (substr($Range, 0, 1) . '-' . substr($Range, -1, 1)); 138 } 139 } 140 /* Not a letter. Check for numbers */ 141 $Find = strpos($Numbers, $Name); 142 if ($Find !== false) { 143 return ("0-9"); 144 } 145 /* Not a letter. */ 146 return ("Other"); 147} /* GetBucketFolder() */ 148 149/** 150 * \brief Given a folder path, return the folder_pk. 151 * 152 * \param $FolderPath - path from -f 153 * \param $Parent - parent folder of $FolderPath 154 * 155 * \return folder_pk, 1: 'Software Repository', others: specified folder 156 157 * \note If any part of the folder path does not exist, thenscp cp2foss will create it. 158 * This is recursive! 159 */ 160function GetFolder($FolderPath, $Parent = null) 161{ 162 $dbManager = $GLOBALS['container']->get('db.manager'); 163 global $Verbose; 164 global $Test; 165 if (empty($Parent)) { 166 $Parent = FolderGetTop(); 167 } 168 /*/ indicates it's the root folder. Empty folder path ends recursion. */ 169 if ($FolderPath == '/') { 170 return ($Parent); 171 } 172 if (empty($FolderPath)) { 173 return ($Parent); 174 } 175 list($folderHead, $folderTail) = explode('/', $FolderPath, 2); 176 if (empty($folderHead)) { 177 return (GetFolder($folderTail, $Parent)); 178 } 179 /* See if it exists */ 180 $SQL = "SELECT folder_pk FROM folder 181 INNER JOIN foldercontents ON child_id = folder_pk 182 AND foldercontents_mode = '1' 183 WHERE foldercontents.parent_fk = $1 AND folder_name = $2"; 184 if ($Verbose) { 185 print "SQL=\n$SQL\n$1=$Parent\n$2=$folderHead\n"; 186 } 187 188 $row = $dbManager->getSingleRow($SQL, array($Parent, $folderHead), __METHOD__.".GetFolder.exists"); 189 if (empty($row)) { 190 /* Need to create folder */ 191 global $Plugins; 192 $P = & $Plugins[plugin_find_id("folder_create") ]; 193 if (empty($P)) { 194 print "FATAL: Unable to find folder_create plugin.\n"; 195 exit(1); 196 } 197 if ($Verbose) { 198 print "Folder not found: Creating $folderHead\n"; 199 } 200 if (!$Test) { 201 $P->create($Parent, $folderHead, ""); 202 $row = $dbManager->getSingleRow($SQL, array($Parent, $folderHead), __METHOD__.".GetFolder.exists"); 203 } 204 } 205 $Parent = $row['folder_pk']; 206 return (GetFolder($folderTail, $Parent)); 207} /* GetFolder() */ 208 209/** 210 * \brief Given one object (file or URL), upload it. 211 * 212 * \param $FolderPath - folder path 213 * \param $UploadArchive - upload file(absolute path) or url 214 * \param $UploadName - uploaded file/dir name 215 * \param $UploadDescription - upload description 216 * 217 * \return 1: error, 0: success 218 */ 219function UploadOne($FolderPath, $UploadArchive, $UploadName, $UploadDescription, $TarSource = null) 220{ 221 $dbManager = $GLOBALS['container']->get('db.manager'); 222 global $Verbose; 223 global $Test; 224 global $QueueList; 225 global $fossjobs_command; 226 global $public_flag; 227 global $SysConf; 228 global $VCS; 229 global $vcsuser; 230 global $vcspass; 231 global $TarExcludeList; 232 global $scmarg; 233 $jobqueuepk = 0; 234 235 if (empty($UploadName)) { 236 $text = "UploadName is empty\n"; 237 echo $text; 238 return 1; 239 } 240 241 $user_pk = $SysConf['auth']['UserId']; 242 $group_pk = $SysConf['auth']['GroupId']; 243 /* Get the user record and check the PLUGIN_DB_ level to make sure they have at least write access */ 244 $UsersRow = GetSingleRec("users", "where user_pk=$user_pk"); 245 if ($UsersRow["user_perm"] < PLUGIN_DB_WRITE) { 246 print "You have no permission to upload files into FOSSology\n"; 247 return 1; 248 } 249 250 /* Get the folder's primary key */ 251 $root_folder_fk = $UsersRow["root_folder_fk"]; 252 global $OptionA; /* Should it use bucket names? */ 253 if ($OptionA) { 254 global $bucket_size; 255 $FolderPath.= "/" . GetBucketFolder($UploadName, $bucket_size); 256 } 257 $FolderPk = GetFolder($FolderPath, $root_folder_fk); 258 if ($FolderPk == 1) { 259 print " Uploading to folder: 'Software Repository'\n"; 260 } else { 261 print " Uploading to folder: '$FolderPath'\n"; 262 } 263 print " Uploading as '$UploadName'\n"; 264 if (!empty($UploadDescription)) { 265 print " Upload description: '$UploadDescription'\n"; 266 } 267 268 $Mode = (1 << 3); // code for "it came from web upload" 269 270 /* Create the upload for the file */ 271 if ($Verbose) { 272 print "JobAddUpload($user_pk, $group_pk, $UploadName,$UploadArchive,$UploadDescription,$Mode,$FolderPk, $public_flag);\n"; 273 } 274 if (!$Test) { 275 $Src = $UploadArchive; 276 if (!empty($TarSource)) { 277 $Src = $TarSource; 278 } 279 $UploadPk = JobAddUpload($user_pk, $group_pk, $UploadName, $Src, $UploadDescription, $Mode, $FolderPk, $public_flag); 280 print " UploadPk is: '$UploadPk'\n"; 281 print " FolderPk is: '$FolderPk'\n"; 282 } 283 284 /* Prepare the job: job "wget" */ 285 if ($Verbose) { 286 print "JobAddJob($user_pk, $group_pk, wget, $UploadPk);\n"; 287 } 288 if (!$Test) { 289 $jobpk = JobAddJob($user_pk, $group_pk, "wget", $UploadPk); 290 if (empty($jobpk) || ($jobpk < 0)) { 291 $text = _("Failed to insert job record"); 292 echo $text; 293 return 1; 294 } 295 } 296 297 $jq_args = "$UploadPk - $Src"; 298 if ($TarExcludeList) { 299 $jq_args .= " " . $TarExcludeList; 300 } 301 if ($VCS) { 302 $jq_args .= " " . $VCS; // add flags when upload from version control system 303 } 304 if ($vcsuser && $vcspass) { 305 $jq_args .= " --username $vcsuser --password $vcspass "; 306 } 307 if ($Verbose) { 308 print "JobQueueAdd($jobpk, wget_agent, $jq_args, no, NULL);\n"; 309 } 310 if (!$Test) { 311 $jobqueuepk = JobQueueAdd($jobpk, "wget_agent", $jq_args, "no", NULL); 312 if (empty($jobqueuepk)) { 313 $text = _("Failed to insert task 'wget' into job queue"); 314 echo $text; 315 return 1; 316 } 317 } 318 /* schedule agents */ 319 global $Plugins; 320 if ($Verbose) { 321 print "AgentAdd wget_agent and dj2nest.\n"; 322 } 323 if (!$Test) { 324 $unpackplugin = &$Plugins[plugin_find_id("agent_unpack") ]; 325 $ununpack_jq_pk = $unpackplugin->AgentAdd($jobpk, $UploadPk, $ErrorMsg, array("wget_agent"), $scmarg); 326 if ($ununpack_jq_pk < 0) { 327 echo $ErrorMsg; 328 return 1; 329 } 330 331 $adj2nestplugin = &$Plugins[plugin_find_id("agent_adj2nest") ]; 332 $adj2nest_jq_pk = $adj2nestplugin->AgentAdd($jobpk, $UploadPk, $ErrorMsg, array()); 333 if ($adj2nest_jq_pk < 0) { 334 echo $ErrorMsg; 335 return 1; 336 } 337 } 338 if (!empty($QueueList)) { 339 switch ($QueueList) { 340 case 'ALL': 341 case 'all': 342 $Cmd = "$fossjobs_command -U '$UploadPk'"; 343 break; 344 default: 345 $Cmd = "$fossjobs_command -U '$UploadPk' -A '$QueueList'"; 346 break; 347 } 348 if ($Verbose) { 349 print "CMD=$Cmd\n"; 350 } 351 if (!$Test) { 352 system($Cmd); 353 } 354 } else { 355 /* No other agents other than unpack scheduled, attach to unpack*/ 356 } 357 global $OptionS; /* Should it run synchronously? */ 358 if ($OptionS) { 359 $working = true; 360 $waitCount = 0; 361 while ($working && ($waitCount++ < 30)) { 362 sleep(3); 363 $SQL = "select 1 from jobqueue inner join job on job.job_pk = jobqueue.jq_job_fk where job_upload_fk = $1 and jq_end_bits = 0 and jq_type = 'wget_agent'"; 364 365 $row = $dbManager->getSingleRow($SQL, array($UploadPk), __METHOD__.".UploadOne"); 366 if (empty($row)) { 367 $working = false; 368 } 369 370 } 371 if ($working) { 372 echo "Gave up waiting for copy completion. Is the scheduler running?"; 373 return 1; 374 } 375 } 376} /* UploadOne() */ 377 378 379/************************************************************************/ 380/************************************************************************/ 381/************************************************************************/ 382/* Process each parameter */ 383$FolderPath = "/"; 384$UploadDescription = ""; 385$UploadName = ""; 386$QueueList = ""; 387$TarExcludeList = ""; 388$bucket_size = 3; 389$public_flag = 0; 390$scmarg = NULL; 391$OptionS = ""; 392 393$user = $passwd = ""; 394$group = ""; 395$vcsuser = $vcspass= ""; 396 397for ($i = 1; $i < $argc; $i ++) { 398 switch ($argv[$i]) { 399 case '-c': 400 $i++; 401 break; /* handled in fo_wrapper */ 402 case '-h': 403 case '-?': 404 print $Usage . "\n"; 405 exit(0); 406 case '--username': 407 $i++; 408 $user = $argv[$i]; 409 break; 410 case '--groupname': 411 $i++; 412 $group = $argv[$i]; 413 break; 414 case '--password': 415 $i++; 416 $passwd = $argv[$i]; 417 break; 418 case '--user': 419 $i++; 420 $vcsuser = $argv[$i]; 421 break; 422 case '--pass': 423 $i++; 424 $vcspass = $argv[$i]; 425 break; 426 case '-A': /* use alphabet buckets */ 427 $OptionA = true; 428 break; 429 case '-AA': /* use alphabet buckets */ 430 $OptionA = true; 431 $i++; 432 $bucket_size = intval($argv[$i]); 433 if ($bucket_size < 1) { 434 $bucket_size = 1; 435 } 436 break; 437 case '-f': /* folder path */ 438 case '-p': /* deprecated 'path' to folder */ 439 $i++; 440 $FolderPath = $argv[$i]; 441 /* idiot check for absolute paths */ 442 //print " Before Idiot Checks: '$FolderPath'\n"; 443 /* remove starting and ending / */ 444 $FolderPath = preg_replace('@^/*@', "", $FolderPath); 445 $FolderPath = preg_replace('@/*$@', "", $FolderPath); 446 /* Note: the pattern below should probably be generalized to remove everything 447 * up to and including the 1st /, This pattern works in what I've 448 * tested: @^.*\/@ ( I had to escape the / so the comment works!) 449 */ 450 $FolderPath = preg_replace("@^S.*? Repository@", "", $FolderPath); 451 $FolderPath = preg_replace('@//*@', "/", $FolderPath); 452 $FolderPath = '/' . $FolderPath; 453 //print " AFTER Idiot Checks: '$FolderPath'\n"; 454 455 break; 456 case '-R': /* obsolete: recurse directories */ 457 break; 458 case '-W': /* obsolete: webserver */ 459 break; 460 case '-w': /* obsolete: URL switch to use wget */ 461 break; 462 case '-d': /* specify upload description */ 463 $i++; 464 $UploadDescription = $argv[$i]; 465 break; 466 case '-n': /* specify upload name */ 467 $i++; 468 $UploadName = $argv[$i]; 469 break; 470 case '-Q': /** list all available processing agents */ 471 $OptionQ = 1; 472 break; 473 case '-q': 474 $i++; 475 $QueueList = $argv[$i]; 476 break; 477 case '-s': 478 $OptionS = true; 479 break; 480 case '-T': /* Test mode */ 481 $Test = 1; 482 if (!$Verbose) { 483 $Verbose++; 484 } 485 break; 486 case '-v': 487 $Verbose++; 488 break; 489 case '-X': 490 if (!empty($TarExcludeList)) { 491 $TarExcludeList .= " "; 492 } 493 $i++; 494 $TarExcludeList .= "--exclude '" . $argv[$i] . "'"; 495 break; 496 case '-a': /* it's an archive! */ 497 /* ignore -a since the next name is a file. */ 498 break; 499 case '-': /* it's an archive list from stdin! */ 500 $stdin_flag = 1; 501 break; 502 case '-P': /* set the permission to public or not */ 503 $i++; 504 if (1 == $argv[$i]) { 505 $public_flag = 1; 506 } else { 507 $public_flag = 0; 508 } 509 break; 510 case '-S': /* upload from subversion repo */ 511 $VCS = 'SVN'; 512 break; 513 case '-G': /* upload from git repo */ 514 $VCS = 'Git'; 515 break; 516 case '-I': /* ignore scm data when scanning */ 517 $scmarg = '-I'; 518 break; 519 default: 520 if (substr($argv[$i], 0, 1) == '-') { 521 print "Unknown parameter: '" . $argv[$i] . "'\n"; 522 print $Usage . "\n"; 523 exit(1); 524 } 525 /* No hyphen means it is a file! */ 526 $UploadArchive = $argv[$i]; 527 } /* switch */ 528} /* for each parameter */ 529 530account_check($user, $passwd, $group); // check username/password 531 532/** list all available processing agents */ 533if (!$Test && $OptionQ) { 534 $Cmd = "fossjobs --username $user --groupname $group --password $passwd -c $SYSCONFDIR -a"; 535 system($Cmd); 536 exit(0); 537} 538 539/** get archive from stdin */ 540if ($stdin_flag) { 541 $Fin = fopen("php://stdin", "r"); 542 if (!feof($Fin)) { 543 $UploadArchive = trim(fgets($Fin)); 544 } 545 fclose($Fin); 546} 547 548/** compose fossjobs command */ 549if ($Verbose) { 550 $fossjobs_command = "fossjobs --username $user --groupname $group --password $passwd -c $SYSCONFDIR -v "; 551} else { 552 $fossjobs_command = "fossjobs --username $user --groupname $group --password $passwd -c $SYSCONFDIR "; 553} 554 555//print "fossjobs_command is:$fossjobs_command\n"; 556 557if (!$UploadArchive) { // upload is empty 558 print "FATAL: No files to upload were specified.\n"; 559 exit(1); 560} 561 562/** get real path, and file name */ 563$UploadArchiveTmp = ""; 564$UploadArchiveTmp = realpath($UploadArchive); 565if (!$UploadArchiveTmp) { 566 // neither a file nor folder from server? 567 if (filter_var($UploadArchive, FILTER_VALIDATE_URL)) { 568 } else if (strchr($UploadArchive, '*')) { 569 $file_number_cmd = "ls $UploadArchive > /dev/null"; 570 system($file_number_cmd, $return_val); 571 if ($return_val) { 572 exit(1); // not files matched 573 } 574 if ("/" != $UploadArchive[0]) { // it is a absolute path 575 $UploadArchive = getcwd()."/".$UploadArchive; 576 } 577 } else { 578 print "Note: it seems that what you want to upload '$UploadArchive' does not exist. \n"; 579 exit(1); 580 } 581} else { // is a file or folder from server 582 $UploadArchive = $UploadArchiveTmp; 583} 584 585if (strlen($UploadArchive) > 0 && empty($UploadName)) { 586 $UploadName = basename($UploadArchive); 587} 588 589if ($vcsuser && $vcspass) { 590 print "Warning: usernames and passwords on the command line are visible to system users with a shell account. To avoid this you can download your source, then upload.\n"; 591} 592 593print "Loading '$UploadArchive'\n"; 594print " Calling UploadOne in 'main': '$FolderPath'\n"; 595$res = UploadOne($FolderPath, $UploadArchive, $UploadName, $UploadDescription); 596if ($res) { 597 exit(1); // fail to upload 598} 599exit(0); 600