diff --git a/src/lib/php/Data/License.php b/src/lib/php/Data/License.php
index 4921214b..e1dd51ae 100644
--- a/src/lib/php/Data/License.php
+++ b/src/lib/php/Data/License.php
@@ -1,87 +1,87 @@
text = $text;
$this->url = $url;
$this->risk = $risk;
$this->spdxCompatible = $spdxCompatible;
}
/**
* @return int
*/
public function getRisk()
{
return $this->risk;
}
/**
* @return boolean
*/
public function getSpdxCompatible()
{
return $this->spdxCompatible;
}
/**
* @return string
*/
public function getText()
{
return $this->text;
}
/**
* @return string
*/
public function getUrl()
{
return $this->url;
}
/** @return LicenseRef */
public function getRef()
{
return new parent($this->getId(), $this->getShortName(), $this->getFullName());
}
}
diff --git a/src/spdx2/agent/spdx2.php b/src/spdx2/agent/spdx2.php
index 6a733af0..6f32259a 100644
--- a/src/spdx2/agent/spdx2.php
+++ b/src/spdx2/agent/spdx2.php
@@ -1,675 +1,722 @@
'N', 'monk' => 'M');
/** @var array */
protected $includedLicenseIds = array();
/** @var string */
+ protected $filebasename = null;
+ /** @var string */
protected $uri;
/** @var string */
+ protected $filename;
+ /** @var string */
protected $outputFormat = self::DEFAULT_OUTPUT_FORMAT;
+ /** @var callable */
+ protected $spdxValidityChecker = null;
+
function __construct()
{
parent::__construct('spdx2', AGENT_VERSION, AGENT_REV);
$this->uploadDao = $this->container->get('dao.upload');
$this->clearingDao = $this->container->get('dao.clearing');
$this->licenseDao = $this->container->get('dao.license');
$this->dbManager = $this->container->get('db.manager');
$this->renderer = $this->container->get('twig.environment');
$this->renderer->setCache(false);
$this->agentSpecifLongOptions[] = self::UPLOAD_ADDS.':';
$this->agentSpecifLongOptions[] = self::OUTPUT_FORMAT_KEY.':';
+
+ $dbManager = $this->dbManager;
+ $licenseDao = $this->licenseDao;
+ $groupId = $this->groupId;
+ $this->spdxValidityChecker = function ($licenseShortname) use ($dbManager, $licenseDao, $groupId) {
+ $lic = $licenseDao->getLicenseByShortName($licenseShortname, $groupId);
+ if ($lic === null)
+ {
+ return false;
+ }
+ return $dbManager->booleanFromDb($lic->getSpdxCompatible());
+ };
}
/**
* @param string[] $args
*
* @return string[] $args
*/
protected function preWorkOnArgs($args)
{
if ((!array_key_exists(self::OUTPUT_FORMAT_KEY,$args)
|| $args[self::OUTPUT_FORMAT_KEY] === "")
&& array_key_exists(self::UPLOAD_ADDS,$args))
{
$args = SpdxTwoUtils::preWorkOnArgsFlp($args,self::UPLOAD_ADDS,self::OUTPUT_FORMAT_KEY);
}
else
{
if (!array_key_exists(self::UPLOAD_ADDS,$args) || $args[self::UPLOAD_ADDS] === "")
{
$args = SpdxTwoUtils::preWorkOnArgsFlp($args,self::UPLOAD_ADDS,self::OUTPUT_FORMAT_KEY);
}
}
return $args;
}
/**
* @param int $uploadId
* @return bool
*/
function processUploadId($uploadId)
{
$args = $this->preWorkOnArgs($this->args);
if(array_key_exists(self::OUTPUT_FORMAT_KEY,$args))
{
$possibleOutputFormat = trim($args[self::OUTPUT_FORMAT_KEY]);
if(in_array($possibleOutputFormat, explode(',',self::AVAILABLE_OUTPUT_FORMATS)))
{
$this->outputFormat = $possibleOutputFormat;
}
}
$this->licenseMap = new LicenseMap($this->dbManager, $this->groupId, LicenseMap::REPORT, true);
$this->computeUri($uploadId);
$packageNodes = $this->renderPackage($uploadId);
$additionalUploadIds = array_key_exists(self::UPLOAD_ADDS,$args) ? explode(',',$args[self::UPLOAD_ADDS]) : array();
$packageIds = array($uploadId);
foreach($additionalUploadIds as $additionalId)
{
$packageNodes .= $this->renderPackage($additionalId);
$packageIds[] = $additionalId;
}
$this->writeReport($packageNodes, $packageIds, $uploadId);
return true;
}
/**
* @param string $partname
* @return string
*/
protected function getTemplateFile($partname)
{
$prefix = $this->outputFormat . "-";
$postfix = ".twig";
switch ($this->outputFormat)
{
case "spdx2":
$postfix = ".xml" . $postfix;
break;
case "spdx2tv":
break;
case "dep5":
$prefix = $prefix . "copyright-";
break;
}
return $prefix . $partname . $postfix;
}
+ protected function getFileBasename($packageName)
+ {
+ if($this->filebasename == null) {
+ $fileName = strtoupper($this->outputFormat)."_".$packageName.'_'.time();
+ switch ($this->outputFormat)
+ {
+ case "spdx2":
+ $fileName = $fileName .".rdf";
+ break;
+ case "spdx2tv":
+ $fileName = $fileName .".spdx";
+ break;
+ case "dep5":
+ $fileName = $fileName .".txt";
+ break;
+ }
+ $this->filebasename = $fileName;
+ }
+ return $this->filebasename;
+ }
+
+ protected function getFileName($packageName)
+ {
+ global $SysConf;
+ $fileBase = $SysConf['FOSSOLOGY']['path']."/report/";
+ return $fileBase. $this->getFileBasename($packageName);
+ }
+
/**
* @param string $fileBase
* @param string $packageName
* @return string
*/
- protected function getUri($fileBase,$packageName)
+ protected function getUri($packageName)
{
- $fileName = $fileBase. strtoupper($this->outputFormat)."_".$packageName.'_'.time();
- switch ($this->outputFormat)
+ global $SysConf;
+ $url=$SysConf['SYSCONFIG']['FOSSologyURL'];
+ if (substr( $url, 0, 4 ) !== "http")
{
- case "spdx2":
- $fileName = $fileName .".rdf" ;
- break;
- case "spdx2tv":
- $fileName = $fileName .".spdx" ;
- break;
- case "dep5":
- $fileName = $fileName .".txt" ;
- break;
+ $url="http://".$url;
}
- return $fileName;
+
+ return $url . $this->getFileBasename($packageName);
}
/**
* @param int $uploadId
* @return string
*/
protected function renderPackage($uploadId)
{
$uploadTreeTableName = $this->uploadDao->getUploadtreeTableName($uploadId);
$itemTreeBounds = $this->uploadDao->getParentItemBounds($uploadId,$uploadTreeTableName);
$this->heartbeat(0);
$filesWithLicenses = $this->getFilesWithLicensesFromClearings($itemTreeBounds);
$this->heartbeat(0);
$this->addClearingStatus($filesWithLicenses,$itemTreeBounds);
$this->heartbeat(0);
$licenseComment = $this->addScannerResults($filesWithLicenses, $itemTreeBounds);
$this->heartbeat(0);
$this->addCopyrightResults($filesWithLicenses, $uploadId);
$this->heartbeat(0);
$upload = $this->uploadDao->getUpload($uploadId);
$fileNodes = $this->generateFileNodes($filesWithLicenses, $upload->getTreeTableName());
$mainLicenseIds = $this->clearingDao->getMainLicenseIds($uploadId, $this->groupId);
$mainLicenses = array();
foreach($mainLicenseIds as $licId)
{
$reportedLicenseId = $this->licenseMap->getProjectedId($licId);
$this->includedLicenseIds[$reportedLicenseId] = true;
$mainLicenses[] = $this->licenseMap->getProjectedShortname($reportedLicenseId);
}
+ if (strcmp($this->outputFormat, "dep5")!==0) {
+ $mainLicenses = SpdxTwoUtils::addPrefixOnDemandList($mainLicenses, $this->spdxValidityChecker);
+ }
+
$hashes = $this->uploadDao->getUploadHashes($uploadId);
return $this->renderString($this->getTemplateFile('package'),array(
'uploadId'=>$uploadId,
'uri'=>$this->uri,
'packageName'=>$upload->getFilename(),
'uploadName'=>$upload->getFilename(),
'sha1'=>$hashes['sha1'],
'md5'=>$hashes['md5'],
'verificationCode'=>$this->getVerificationCode($upload),
'mainLicenses'=>$mainLicenses,
- 'mainLicense'=>SpdxTwoUtils::implodeLicenses($mainLicenses, "LicenseRef-"),
+ 'mainLicense'=>SpdxTwoUtils::implodeLicenses($mainLicenses, $this->spdxValidityChecker),
'licenseComments'=>$licenseComment,
'fileNodes'=>$fileNodes)
);
}
/**
* @param ItemTreeBounds $itemTreeBounds
* @return string[][][] $filesWithLicenses mapping item->'concluded'->(array of shortnames)
*/
protected function getFilesWithLicensesFromClearings(ItemTreeBounds $itemTreeBounds)
{
$clearingDecisions = $this->clearingDao->getFileClearingsFolder($itemTreeBounds, $this->groupId);
$filesWithLicenses = array();
$clearingsProceeded = 0;
foreach ($clearingDecisions as $clearingDecision)
{
$clearingsProceeded += 1;
if(($clearingsProceeded&2047)==0)
{
$this->heartbeat(0);
}
if($clearingDecision->getType() == DecisionTypes::IRRELEVANT)
{
continue;
}
foreach ($clearingDecision->getClearingEvents() as $clearingEvent)
{
$clearingLicense = $clearingEvent->getClearingLicense();
if ($clearingLicense->isRemoved())
{
continue;
}
- $spdxCheck = $this->dbManager->booleanFromDb($this->licenseDao->getLicenseById($clearingLicense->getLicenseId(), $this->groupId)->getSpdxCompatible());
- if($spdxCheck){
- $prefix = "";
- }else{
- $prefix = "LicenseRef-";
- }
if($clearingEvent->getReportinfo())
{
$customLicenseText = $clearingEvent->getReportinfo();
$reportedLicenseShortname = $this->licenseMap->getProjectedShortname($this->licenseMap->getProjectedId($clearingLicense->getLicenseId())) .
'-' . md5($customLicenseText);
$this->includedLicenseIds[$reportedLicenseShortname] = $customLicenseText;
$filesWithLicenses[$clearingDecision->getUploadTreeId()]['concluded'][] = $reportedLicenseShortname;
- $filesWithLicenses[$clearingDecision->getUploadTreeId()]['prefix'][$reportedLicenseShortname] = $prefix;
}
else
{
$reportedLicenseId = $this->licenseMap->getProjectedId($clearingLicense->getLicenseId());
$this->includedLicenseIds[$reportedLicenseId] = true;
$filesWithLicenses[$clearingDecision->getUploadTreeId()]['concluded'][] = $this->licenseMap->getProjectedShortname($reportedLicenseId);
- $filesWithLicenses[$clearingDecision->getUploadTreeId()]['prefix'][$this->licenseMap->getProjectedShortname($reportedLicenseId)] = $prefix;
}
}
}
return $filesWithLicenses;
}
/**
* @param string[][][] $filesWithLicenses
* @param string[] $licenses
* @param string[] $copyrights
* @param string $file
* @param string $fullPath
*/
protected function toLicensesWithFilesAdder(&$filesWithLicenses, $licenses, $copyrights, $file, $fullPath)
{
$key = SpdxTwoUtils::implodeLicenses($licenses);
if (!array_key_exists($key, $filesWithLicenses))
{
$filesWithLicenses[$key]['files']=array();
$filesWithLicenses[$key]['copyrights']=array();
}
$filesWithLicenses[$key]['files'][$file] = $fullPath;
foreach ($copyrights as $copyright) {
if (!in_array($copyright, $filesWithLicenses[$key]['copyrights'])) {
$filesWithLicenses[$key]['copyrights'][] = $copyright;
}
}
}
/**
* @param string[][][] $filesWithLicenses
* @param string $treeTableName
*/
protected function toLicensesWithFiles(&$filesWithLicenses, $treeTableName)
{
$licensesWithFiles = array();
$treeDao = $this->container->get('dao.tree');
$filesProceeded = 0;
foreach($filesWithLicenses as $fileId=>$licenses)
{
$filesProceeded += 1;
if(($filesProceeded&2047)==0)
{
$this->heartbeat(0);
}
$fullPath = $treeDao->getFullPath($fileId,$treeTableName);
if(!empty($licenses['concluded']) && count($licenses['concluded'])>0)
{
$this->toLicensesWithFilesAdder($licensesWithFiles,$licenses['concluded'],$licenses['copyrights'],$fileId,$fullPath);
}
else
{
if(!empty($licenses['scanner']) && count($licenses['scanner'])>0)
{
$implodedLicenses = SpdxTwoUtils::implodeLicenses($licenses['scanner']);
if($licenses['isCleared'])
{
$msgLicense = "None (scanners found: " . $implodedLicenses . ")";
}
else
{
$msgLicense = "NoLicenseConcluded (scanners found: " . $implodedLicenses . ")";
}
}
else
{
if($licenses['isCleared'])
{
$msgLicense = "None";
}
else
{
$msgLicense = "NoLicenseConcluded";
}
}
$this->toLicensesWithFilesAdder($licensesWithFiles,array($msgLicense),$licenses['copyrights'],$fileId,$fullPath);
}
}
return $licensesWithFiles;
}
/**
* @param string[][][] &$filesWithLicenses
* @param ItemTreeBounds $itemTreeBounds
*/
protected function addScannerResults(&$filesWithLicenses, ItemTreeBounds $itemTreeBounds)
{
$uploadId = $itemTreeBounds->getUploadId();
$scannerAgents = array_keys($this->agentNames);
$scanJobProxy = new ScanJobProxy($this->container->get('dao.agent'), $uploadId);
$scanJobProxy->createAgentStatus($scannerAgents);
$scannerIds = $scanJobProxy->getLatestSuccessfulAgentIds();
if(empty($scannerIds))
{
- return;
+ return "";
}
$selectedScanners = '{'.implode(',',$scannerIds).'}';
$tableName = $itemTreeBounds->getUploadTreeTableName();
$stmt = __METHOD__ .'.scanner_findings';
$sql = "SELECT DISTINCT uploadtree_pk,rf_fk FROM $tableName ut, license_file
WHERE ut.pfile_fk=license_file.pfile_fk AND rf_fk IS NOT NULL AND agent_fk=any($1)";
$param = array($selectedScanners);
if ($tableName == 'uploadtree_a') {
$param[] = $uploadId;
$sql .= " AND upload_fk=$".count($param);
$stmt .= $tableName;
}
$sql .= " GROUP BY uploadtree_pk,rf_fk";
$this->dbManager->prepare($stmt, $sql);
$res = $this->dbManager->execute($stmt,$param);
while($row=$this->dbManager->fetchArray($res))
{
$reportedLicenseId = $this->licenseMap->getProjectedId($row['rf_fk']);
$shortName = $this->licenseMap->getProjectedShortname($reportedLicenseId);
if ($shortName != 'No_license_found' && $shortName != 'Void') {
- $filesWithLicenses[$row['uploadtree_pk']]['scanner'][] = $shortName;
$this->includedLicenseIds[$reportedLicenseId] = true;
}
}
$this->dbManager->freeResult($res);
return "licenseInfoInFile determined by Scanners $selectedScanners";
}
/**
* @param string[][][] &$filesWithLicenses
* @param int $uploadId
* @return string
*/
protected function addCopyrightResults(&$filesWithLicenses, $uploadId)
{
/* @var $copyrightDao CopyrightDao */
$copyrightDao = $this->container->get('dao.copyright');
$uploadtreeTable = $this->uploadDao->getUploadtreeTableName($uploadId);
$allEntries = $copyrightDao->getAllEntries('copyright', $uploadId, $uploadtreeTable, $type='statement');
foreach ($allEntries as $finding) {
$filesWithLicenses[$finding['uploadtree_pk']]['copyrights'][] = \convertToUTF8($finding['content'],false);
}
}
/**
* @param string[][][] &$filesWithLicenses
* @param ItemTreeBounds $itemTreeBounds
*/
protected function addClearingStatus(&$filesWithLicenses,ItemTreeBounds $itemTreeBounds)
{
$alreadyClearedUploadTreeView = new UploadTreeProxy($itemTreeBounds->getUploadId(),
array(UploadTreeProxy::OPT_SKIP_THESE => UploadTreeProxy::OPT_SKIP_ALREADY_CLEARED,
UploadTreeProxy::OPT_ITEM_FILTER => "AND (lft BETWEEN ".$itemTreeBounds->getLeft()." AND ".$itemTreeBounds->getRight().")",
UploadTreeProxy::OPT_GROUP_ID => $this->groupId),
$itemTreeBounds->getUploadTreeTableName(),
'already_cleared_uploadtree' . $itemTreeBounds->getUploadId());
$alreadyClearedUploadTreeView->materialize();
$filesThatShouldStillBeCleared = $alreadyClearedUploadTreeView->getNonArtifactDescendants($itemTreeBounds);
$alreadyClearedUploadTreeView->unmaterialize();
$uploadTreeIds = array_keys($filesWithLicenses);
foreach($uploadTreeIds as $uploadTreeId)
{
$filesWithLicenses[$uploadTreeId]['isCleared'] = false == array_key_exists($uploadTreeId,$filesThatShouldStillBeCleared);
}
}
/**
* @param int $uploadId
*/
protected function computeUri($uploadId)
{
- global $SysConf;
$upload = $this->uploadDao->getUpload($uploadId);
$packageName = $upload->getFilename();
- $fileBase = $SysConf['FOSSOLOGY']['path']."/report/";
-
- $this->uri = $this->getUri($fileBase,$packageName);
+ $this->uri = $this->getUri($packageName);
+ $this->filename = $this->getFileName($packageName);
}
/**
- * @param string[] $packageNodes
+ * @param string $packageNodes
* @param int[] $packageIds
* @param int $uploadId
*/
protected function writeReport(&$packageNodes, $packageIds, $uploadId)
{
- $fileBase = dirname($this->uri);
+ $fileBase = dirname($this->filename);
if(!is_dir($fileBase)) {
mkdir($fileBase, 0777, true);
}
umask(0133);
+ $licenseTexts=$this->getLicenseTexts();
+ if (strcmp($this->outputFormat, "dep5")!==0) {
+ $licenseTexts = SpdxTwoUtils::addPrefixOnDemandKeys($licenseTexts, $this->spdxValidityChecker);
+ }
+
$message = $this->renderString($this->getTemplateFile('document'),array(
'documentName'=>$fileBase,
'uri'=>$this->uri,
'userName'=>$this->container->get('dao.user')->getUserName($this->userId),
'organisation'=>'',
'packageNodes'=>$packageNodes,
'packageIds'=>$packageIds,
- 'licenseTexts'=>$this->getLicenseTexts())
+ 'licenseTexts'=>$licenseTexts)
);
// To ensure the file is valid, replace any non-printable characters with a question mark.
// 'Non-printable' is ASCII < 0x20 (excluding \r, \n and tab) and 0x7F (delete).
$message = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/','?',$message);
- file_put_contents($this->uri, $message);
- $this->updateReportTable($uploadId, $this->jobId, $this->uri);
+ file_put_contents($this->filename, $message);
+ $this->updateReportTable($uploadId, $this->jobId, $this->filename);
}
/**
* @param int $uploadId
* @param int $jobId
* @param string $fileName
*/
protected function updateReportTable($uploadId, $jobId, $fileName){
$this->dbManager->insertTableRow('reportgen',
array('upload_fk'=>$uploadId, 'job_fk'=>$jobId, 'filepath'=>$fileName),
__METHOD__);
}
/**
* @param string $templateName
* @param array $vars
* @return string
*/
protected function renderString($templateName, $vars)
{
return $this->renderer->loadTemplate($templateName)->render($vars);
}
protected function generateFileNodes($filesWithLicenses, $treeTableName)
{
if (strcmp($this->outputFormat, "dep5")!==0)
{
return $this->generateFileNodesByFiles($filesWithLicenses, $treeTableName);
}
else
{
return $this->generateFileNodesByLicenses($filesWithLicenses, $treeTableName);
}
}
/**
* @param string[][][] &$filesWithLicenses
* @param string $treeTableName
* @return string
*/
protected function generateFileNodesByFiles($filesWithLicenses, $treeTableName)
{
/* @var $treeDao TreeDao */
$treeDao = $this->container->get('dao.tree');
$filesProceeded = 0;
$lastValue = 0;
$content = '';
foreach($filesWithLicenses as $fileId=>$licenses)
{
$filesProceeded += 1;
if(($filesProceeded&2047)==0)
{
$this->heartbeat($filesProceeded - $lastValue);
$lastValue = $filesProceeded;
}
$hashes = $treeDao->getItemHashes($fileId);
$fileName = $treeDao->getFullPath($fileId,$treeTableName);
+ if(!is_array($licenses['concluded']))
+ {
+ $licenses['concluded'] = array();
+ }
+ if(!is_array($licenses['scanner']))
+ {
+ $licenses['scanner'] = array();
+ }
$content .= $this->renderString($this->getTemplateFile('file'),array(
'fileId'=>$fileId,
'sha1'=>$hashes['sha1'],
'md5'=>$hashes['md5'],
'uri'=>$this->uri,
'fileName'=>$fileName,
'fileDirName'=>dirname($fileName),
'fileBaseName'=>basename($fileName),
'isCleared'=>$licenses['isCleared'],
- 'concludedLicense'=>SpdxTwoUtils::implodeLicenses($licenses['concluded'], $licenses['prefix']),
- 'concludedLicenses'=>$licenses['concluded'],
- 'scannerLicenses'=>$licenses['scanner'],
+ 'concludedLicense'=>SpdxTwoUtils::implodeLicenses($licenses['concluded'], $this->spdxValidityChecker),
+ 'concludedLicenses'=> SpdxTwoUtils::addPrefixOnDemandList($licenses['concluded'], $this->spdxValidityChecker),
+ 'scannerLicenses'=>SpdxTwoUtils::addPrefixOnDemandList($licenses['scanner'], $this->spdxValidityChecker),
'copyrights'=>$licenses['copyrights']));
}
- $this->heartbeat($filesProceeded - $lastValue);
+ $this->heartbeat($filesProceeded - $lastValue);
return $content;
}
/**
* @param string[][][] &$filesWithLicenses
* @param string $treeTableName
* @return string
*/
protected function generateFileNodesByLicenses($filesWithLicenses, $treeTableName)
{
$licensesWithFiles = $this->toLicensesWithFiles($filesWithLicenses, $treeTableName);
$content = '';
$filesProceeded = 0;
$lastStep = 0;
$lastValue = 0;
foreach($licensesWithFiles as $licenseId=>$entry)
{
$filesProceeded += count($entry['files']);
if($filesProceeded&(~2047) > $lastStep)
{
$this->heartbeat($filesProceeded - $lastValue);
$lastStep = $filesProceeded&(~2047) + 2048;
$lastValue = $filesProceeded;
}
$comment = "";
if (strrpos($licenseId, "NoLicenseConcluded (scanners found: ", -strlen($licenseId)) !== false)
{
$comment = substr($licenseId,20,strlen($licenseId)-21);
$licenseId = "NoLicenseConcluded";
}
elseif (strrpos($licenseId, "None (scanners found: ", -strlen($licenseId)) !== false)
{
$comment = substr($licenseId,6,strlen($licenseId)-7);
$licenseId = "None";
}
$content .= $this->renderString($this->getTemplateFile('file'),array(
'fileNames'=>$entry['files'],
'license'=>$licenseId,
'copyrights'=>$entry['copyrights'],
'comment'=>$comment));
}
$this->heartbeat($filesProceeded - $lastValue);
return $content;
}
/**
* @return string[] with keys being shortname
*/
protected function getLicenseTexts() {
$licenseTexts = array();
$licenseViewProxy = new LicenseViewProxy($this->groupId,array(LicenseViewProxy::OPT_COLUMNS=>array('rf_pk','rf_shortname','rf_text')));
$this->dbManager->prepare($stmt=__METHOD__, $licenseViewProxy->getDbViewQuery());
$res = $this->dbManager->execute($stmt);
while($row=$this->dbManager->fetchArray($res))
{
if (array_key_exists($row['rf_pk'], $this->includedLicenseIds))
{
$licenseTexts[$row['rf_shortname']] = $row['rf_text'];
}
}
foreach($this->includedLicenseIds as $license => $customText)
{
if (true !== $customText)
{
$licenseTexts[$license] = $customText;
}
}
$this->dbManager->freeResult($res);
return $licenseTexts;
}
/**
* @param UploadTree $upload
* @return string
*/
protected function getVerificationCode(Upload $upload)
{
$stmt = __METHOD__;
$param = array();
if ($upload->getTreeTableName()=='uploadtree_a')
{
$sql = $upload->getTreeTableName().' WHERE upload_fk=$1 AND';
$param[] = $upload->getId();
}
else
{
$sql = $upload->getTreeTableName().' WHERE';
$stmt .= '.'.$upload->getTreeTableName();
}
$sql = "SELECT STRING_AGG(lower_sha1,'') concat_sha1 FROM
(SELECT LOWER(pfile_sha1) lower_sha1 FROM pfile, $sql pfile_fk=pfile_pk ORDER BY pfile_sha1) templist";
$filelistPack = $this->dbManager->getSingleRow($sql,$param,$stmt);
return sha1($filelistPack['concat_sha1']);
}
}
$agent = new SpdxTwoAgent();
$agent->scheduler_connect();
$agent->run_scheduler_event_loop();
$agent->scheduler_disconnect(0);
diff --git a/src/spdx2/agent/spdx2utils.php b/src/spdx2/agent/spdx2utils.php
index d7b5f2fa..1a096e8d 100644
--- a/src/spdx2/agent/spdx2utils.php
+++ b/src/spdx2/agent/spdx2utils.php
@@ -1,89 +1,125 @@
$text)
+ {
+ $ret[self::addPrefixOnDemand($license, $spdxValidityChecker)] = $text;
+ }
+ return $ret;
+ }
+
+ static public function addPrefixOnDemandList($licenses, $spdxValidityChecker = null)
+ {
+ $ret = array();
+ foreach($licenses as $license)
+ {
+ $ret[] = self::addPrefixOnDemand($license, $spdxValidityChecker);
+ }
+ return $ret;
+ }
+
+ /**
+ * @param string[] $licenses
+ * @param callable $spdxValidityChecker
+ *
+ * @return string
+ */
+ static public function implodeLicenses($licenses, $spdxValidityChecker = null)
+ {
+ if(!$licenses || !is_array($licenses) || sizeof($licenses) == 0)
+ {
+ return "";
+ }
+
+ $licenses = array_map(function ($license) use ($spdxValidityChecker)
+ {
+ return SpdxTwoUtils::addPrefixOnDemand($license, $spdxValidityChecker);
},$licenses);
sort($licenses, SORT_NATURAL | SORT_FLAG_CASE);
if(count($licenses) == 3 &&
- ($index = array_search($prefix . "Dual-license",$licenses)) !== false)
+ ($index = array_search("Dual-license",$licenses)) !== false)
+ {
+ return $licenses[$index===0?1:0] . " OR " . $licenses[$index===2?1:2];
+ }
+ elseif(count($licenses) == 3 &&
+ ($index = array_search(self::$prefix . "Dual-license",$licenses)) !== false)
{
return $licenses[$index===0?1:0] . " OR " . $licenses[$index===2?1:2];
}
else
{
// Add prefixes where needed, enclose statments containing ' OR ' with parantheses
return implode(" AND ", $licenses);
}
}
}
diff --git a/src/spdx2/agent/template/spdx2-document.xml.twig b/src/spdx2/agent/template/spdx2-document.xml.twig
index 6d3460bd..47aef1d6 100644
--- a/src/spdx2/agent/template/spdx2-document.xml.twig
+++ b/src/spdx2/agent/template/spdx2-document.xml.twig
@@ -1,39 +1,43 @@
{# Copyright 2015 Siemens AG
Copying and distribution of this file, with or without modification,
are permitted in any medium without royalty provided the copyright notice and this notice are preserved.
This file is offered as-is, without any warranty.
#}
SPDX-2.0
2.0
Person: {{ userName|e }}
Organization: {{ organisation|e }}
Tool: spxd2
{{ 'now'|date('Y-m-d\\TH:i:s\\Z')}}{# date('c') for ISO 8601 is not parsed by spdxTool #}
{{ documentName|e }}
This document was created using license informations and a generator from Fossology.
- {% for licenseId,extractedText in licenseTexts %}
+{% for licenseId,extractedText in licenseTexts %}
-
- LicenseRef-{{ licenseId|replace(' ','-')|e }}
+{% if licenseId starts with 'LicenseRef-' %}
+
+{% else %}
+
+{% endif %}
+ {{ licenseId|replace(' ','-')|e }}
':']]]]>'}) }}
]]>
- {% endfor %}
+{% endfor %}
{{ packageNodes|replace({'\n':'\n '}) }}
diff --git a/src/spdx2/agent/template/spdx2-file.xml.twig b/src/spdx2/agent/template/spdx2-file.xml.twig
index 35a3359c..b636ba28 100644
--- a/src/spdx2/agent/template/spdx2-file.xml.twig
+++ b/src/spdx2/agent/template/spdx2-file.xml.twig
@@ -1,52 +1,60 @@
{# Copyright 2015 Siemens AG
Copying and distribution of this file, with or without modification,
are permitted in any medium without royalty provided the copyright notice and this notice are preserved.
This file is offered as-is, without any warranty.
#}
{{ fileName|e }}
{{ sha1 }}
{{ md5 }}
{% if isCleared %}
{% if concludedLicenses|default is empty %}
{% else %}
{% for res in concludedLicenses %}
-
+{% if res starts with 'LicenseRef-' %}
+
+{% else %}
+
+{% endif %}
{% endfor %}
{% endif %}
{% else %}
{% endif %}
{% if scannerLicenses|default is empty %}
{% else %}{% for res in scannerLicenses %}
-
+{% if res starts with 'LicenseRef-' %}
+
+{% else %}
+
+{% endif %}
{% endfor %}{% endif %}
{% if copyrights|default is empty %}
{% else %} ':']]]]>'}) }}
{% endfor %}
]]>
{% endif %}
diff --git a/src/spdx2/agent/template/spdx2-package.xml.twig b/src/spdx2/agent/template/spdx2-package.xml.twig
index 5e56a26e..e60a3a06 100644
--- a/src/spdx2/agent/template/spdx2-package.xml.twig
+++ b/src/spdx2/agent/template/spdx2-package.xml.twig
@@ -1,50 +1,54 @@
{# Copyright 2015 Siemens AG
Copying and distribution of this file, with or without modification,
are permitted in any medium without royalty provided the copyright notice and this notice are preserved.
This file is offered as-is, without any warranty.
#}
{{ packageName }}
{{ uploadName }}
{{ verificationCode }}
{{ sha1 }}
{{ md5 }}
{% for res in mainLicenses %}
-
+ {% if res starts with 'LicenseRef-' %}
+
+ {% else %}
+
+ {% endif %}
{% endfor %}
{% if licenseComments %}':']]>'}) }}
]]>
{% endif %}
{{ fileNodes|replace({'\n':'\n '}) }}
diff --git a/src/spdx2/agent/template/spdx2tv-document.twig b/src/spdx2/agent/template/spdx2tv-document.twig
index 0bafe07d..57805348 100644
--- a/src/spdx2/agent/template/spdx2tv-document.twig
+++ b/src/spdx2/agent/template/spdx2tv-document.twig
@@ -1,51 +1,51 @@
{# Copyright 2015 Siemens AG
Copying and distribution of this file, with or without modification,
are permitted in any medium without royalty provided the copyright notice and this notice are preserved.
This file is offered as-is, without any warranty.
#}
SPDXVersion: SPDX-2.0
DataLicense: CC0-1.0
##-------------------------
## Document Information
##-------------------------
DocumentNamespace: {{ uri }}
DocumentName: {{ documentName }}
SPDXID: SPDXRef-DOCUMENT
##-------------------------
## Creation Information
##-------------------------
Creator: Tool: spxd2
Creator: Person: {{ userName }}
Creator: Organization: {{ organisation }}
CreatorComment:
This document was created using license informations and a generator from Fossology.
Created: {{ 'now'|date('Y-m-d\\TH:i:s\\Z')}}
LicenseListVersion: 2.0
##-------------------------
## Package Information
##-------------------------
{% for packageId in packageIds %}
Relationship: SPDXRef-DOCUMENT DESCRIBES SPDXRef-upload{{ packageId }}
{% endfor %}
{{ packageNodes }}
##-------------------------
## License Information
##-------------------------
{% for licenseId,extractedText in licenseTexts %}
-LicenseID: LicenseRef-{{ licenseId|replace(' ','-') }}
+LicenseID: {{ licenseId|replace(' ','-') }}
LicenseName: {{ licenseId|replace(' ','-') }}
ExtractedText: {{ extractedText|replace({'':'<text>','':'</text>'})
|replace({'\f':''}) }}
{% endfor %}
diff --git a/src/spdx2/agent/template/spdx2tv-file.twig b/src/spdx2/agent/template/spdx2tv-file.twig
index 580cb982..f0fc5d89 100644
--- a/src/spdx2/agent/template/spdx2tv-file.twig
+++ b/src/spdx2/agent/template/spdx2tv-file.twig
@@ -1,38 +1,38 @@
{# Copyright 2015 Siemens AG
Copying and distribution of this file, with or without modification,
are permitted in any medium without royalty provided the copyright notice and this notice are preserved.
This file is offered as-is, without any warranty.
#}
##File
FileName: {{ fileName }}
SPDXID: SPDXRef-item{{ fileId }}
{# FileType: OTHER #}
FileChecksum: SHA1: {{ sha1 }}
FileChecksum: MD5: {{ md5 }}
{% if isCleared %}
{% if concludedLicenses|default is empty %}
LicenseConcluded: NONE
{% else %}
LicenseConcluded: {{ concludedLicense }}
{% endif%}
{% else %}
LicenseConcluded: NOASSERTION
{% endif %}
{% if scannerLicenses|default is empty %}
LicenseInfoInFile: NOASSERTION
{% else %}
{% for lic in scannerLicenses %}
-LicenseInfoInFile: LicenseRef-{{ lic|replace({' ':'-'})}}
+LicenseInfoInFile: {{ lic|replace({' ':'-'})}}
{% endfor %}
{% endif %}
{# FileSize: 48 Kb (49173 bytes) #}
{% if copyrights|default is empty %}
FileCopyrightText: NONE
{% else %}
FileCopyrightText: {{ copyrights|join('\n')
|replace({'':'<text>','':'</text>'})
|replace({'\f':''}) }}
{% endif %}
diff --git a/src/spdx2/agent_tests/Functional/schedulerTest.php b/src/spdx2/agent_tests/Functional/schedulerTest.php
index d680baa6..62103e2a 100644
--- a/src/spdx2/agent_tests/Functional/schedulerTest.php
+++ b/src/spdx2/agent_tests/Functional/schedulerTest.php
@@ -1,174 +1,189 @@
testDb = new TestPgDb("spdx2test");
$this->dbManager = $this->testDb->getDbManager();
$this->runnerCli = new SchedulerTestRunnerCli($this->testDb);
$this->assertCountBefore = \Hamcrest\MatcherAssert::getCount();
$this->agentDir = dirname(dirname(__DIR__));
}
protected function tearDown()
{
$this->testDb->fullDestruct();
$this->testDb = null;
$this->dbManager = null;
}
private function setUpRepo()
{
$sysConf = $this->testDb->getFossSysConf();
$this->testInstaller = new TestInstaller($sysConf);
$this->testInstaller->init();
$this->testInstaller->cpRepo();
$this->testInstaller->install($this->agentDir);
}
private function rmRepo()
{
$this->testInstaller->uninstall($this->agentDir);
$this->testInstaller->rmRepo();
$this->testInstaller->clear();
}
private function setUpTables()
{
$this->testDb->createPlainTables(array(),true);
$this->testDb->createInheritedTables();
$this->dbManager->queryOnce("CREATE TABLE copyright_ars () INHERITS (ars_master)");
$this->testDb->createSequences(array('agent_agent_pk_seq','pfile_pfile_pk_seq','upload_upload_pk_seq','nomos_ars_ars_pk_seq','license_file_fl_pk_seq','license_ref_rf_pk_seq','license_ref_bulk_lrb_pk_seq','clearing_decision_clearing_decision_pk_seq','clearing_event_clearing_event_pk_seq'));
$this->testDb->createConstraints(array('agent_pkey','pfile_pkey','upload_pkey_idx','FileLicense_pkey','clearing_event_pkey'));
$this->testDb->alterTables(array('agent','pfile','upload','ars_master','license_ref_bulk','license_set_bulk','clearing_event','clearing_decision','license_file','highlight'));
$this->testDb->insertData(array('mimetype_ars','pkgagent_ars','ununpack_ars','decider_ars'),true,__DIR__.'/fo_report.sql');
$this->testDb->resetSequenceAsMaxOf('agent_agent_pk_seq', 'agent', 'agent_pk');
}
private function getHeartCount($output)
{
$matches = array();
if (preg_match("/.*HEART: ([0-9]*).*/", $output, $matches))
{
return intval($matches[1]);
}
return -1;
}
/** @group Functional */
public function testReportForNormalUploadtreeTable()
{
$this->setUpTables();
$this->setUpRepo();
$this->runAndTestReport();
}
/** @group Functional */
public function testReportForSpecialUploadtreeTable()
{
$this->setUpTables();
$this->setUpRepo();
$uploadId = 1;
$this->dbManager->queryOnce("ALTER TABLE uploadtree_a RENAME TO uploadtree_$uploadId", __METHOD__.'.alterUploadtree');
$this->dbManager->getSingleRow("UPDATE upload SET uploadtree_tablename=$1 WHERE upload_pk=$2",
array("uploadtree_$uploadId",$uploadId),__METHOD__.'.alterUpload');
$this->runAndTestReport($uploadId);
}
public function runAndTestReport($uploadId=1)
{
list($success,$output,$retCode) = $this->runnerCli->run($uploadId, $this->userId, $this->groupId, $jobId=7);
assertThat('cannot run runner', $success, equalTo(true));
- assertThat( 'report failed: "'.$output.'"', $retCode, equalTo(0));
+ assertThat('report failed: "'.$output.'"', $retCode, equalTo(0));
assertThat($this->getHeartCount($output), greaterThan(0));
$row = $this->dbManager->getSingleRow("SELECT upload_fk,job_fk,filepath FROM reportgen WHERE job_fk = $1", array($jobId), "reportFileName");
assertThat($row, hasKeyValuePair('upload_fk', $uploadId));
assertThat($row, hasKeyValuePair('job_fk', $jobId));
$filepath = $row['filepath'];
assertThat($filepath, endsWith('.rdf'));
assertThat(file_exists($filepath),equalTo(true));
$copyrightStatement = 'Copyright (C) 1991-2, RSA Data Security, Inc. Created 1991. All rights reserved';
assertThat(file_get_contents($filepath), stringContainsInOrder($copyrightStatement));
$email = 'condor-admin@cs.wisc.edu';
assertThat(file_get_contents($filepath), not(stringContainsInOrder($email)));
$this->addToAssertionCount(\Hamcrest\MatcherAssert::getCount()-$this->assertCountBefore);
$this->verifyRdf($filepath);
unlink($filepath);
$this->rmRepo();
}
protected function verifyRdf($filepath)
{
$lines = '';
$returnVar = 0;
exec('which java', $lines, $returnVar);
$this->assertEquals(0,$returnVar,'java required for this test');
- $toolJarFile = __DIR__.'/spdx-tools-2.0.2-jar-with-dependencies.jar';
- $this->pullSpdxTools($toolJarFile);
+ $toolJarFile = $this->pullSpdxTools();
$verification = exec("java -jar $toolJarFile Verify $filepath");
assertThat($verification,equalTo('This SPDX Document is valid.'));
}
- protected function pullSpdxTools($jarFile)
+ protected function pullSpdxTools()
{
+ $version='2.1.0';
+ $tag='v'.$version;
+
+ $jarFileBasename = 'spdx-tools-'.$version.'-jar-with-dependencies.jar';
+ $jarFile = __DIR__.'/'.$jarFileBasename;
if(!file_exists($jarFile))
{
- file_put_contents($jarFile, fopen('https://github.com/spdx/tools/releases/download/V2.0.2/spdx-tools-2.0.2-jar-with-dependencies.jar', 'r'));
+ $zipFileBasename='SPDXTools-'.$tag.'.zip';
+ $zipFile=__DIR__.'/'.$zipFileBasename;
+ if(!file_exists($zipFile))
+ {
+ file_put_contents($zipFile, fopen('https://github.com/spdx/tools/releases/download/'.$tag.'/'.$zipFileBasename, 'r'));
+
+ }
+ $this->assertFileExists($zipFile, 'could not download SPDXTools');
+
+ system('unzip -n -d '.__DIR__.' '.$zipFile);
+ rename (__DIR__.'/SPDXTools-'.$tag.'/'.$jarFileBasename, $jarFile);
}
- $this->assertFileExists($jarFile, 'could not download SPDXTools');
+ $this->assertFileExists($jarFile, 'could not extract SPDXTools');
+ return $jarFile;
}
}
diff --git a/src/spdx2/agent_tests/Unit/spdx2utilTest.php b/src/spdx2/agent_tests/Unit/spdx2utilTest.php
index ccb779b0..c68fe124 100644
--- a/src/spdx2/agent_tests/Unit/spdx2utilTest.php
+++ b/src/spdx2/agent_tests/Unit/spdx2utilTest.php
@@ -1,86 +1,91 @@
assertCountBefore = \Hamcrest\MatcherAssert::getCount();
}
protected function tearDown()
{
$this->addToAssertionCount(\Hamcrest\MatcherAssert::getCount()-$this->assertCountBefore);
}
public function testPreWorkOnArgsFlpZero()
{
$args = array();
assertThat(SpdxTwoUtils::preWorkOnArgsFlp($args,"key1","key2"), equalTo($args));
}
public function testPreWorkOnArgsFlpId()
{
$args = array("key1" => "value");
assertThat(SpdxTwoUtils::preWorkOnArgsFlp($args,"key1","key2"), equalTo($args));
}
public function testPreWorkOnArgsFlpRealWork()
{
$args = array("key1" => "value --key2=anotherValue");
$result = SpdxTwoUtils::preWorkOnArgsFlp($args,"key1","key2");
assertThat($result["key1"], equalTo("value"));
assertThat($result["key2"], equalTo("anotherValue"));
}
public function provideLicenseSet()
{
+ $constTrue = function ($licId) { return true; };
+ $constFalse = function ($licId) { return false; };
+
return array(
- 'null' => array(null, '', ''),
- 'empty array' => array(array(), '', ''),
- 'empty array but prefix' => array(array(), 'pre', ''),
- 'single license'=>array(array("LIC1"), '', 'LIC1'),
- 'multiple licenses' => array(array("LIC1","LIC2","LIC3"), '', 'LIC1 AND LIC2 AND LIC3'),
- 'dual license 1st pos' => array(array("Dual-license", "LIC2", "LIC3"), '', 'LIC2 OR LIC3'),
- 'dual license 2nd pos' => array(array("LIC1", "Dual-license", "LIC3"), '', 'LIC1 OR LIC3'),
- 'dual license 3rd pos' => array(array("LIC1", "LIC2", "Dual-license"), '', 'LIC1 OR LIC2'),
- 'dual license with prefix' => array(array("LIC1","LIC2", "Dual-license"), 'pre-', 'pre-LIC1 OR pre-LIC2'),
- 'multiple dualLicense' => array(array("LIC1","LIC2 OR LIC3"), '', '(LIC2 OR LIC3) AND LIC1'),
- 'multiple dualLicense with prefix' => array(array("LIC1","LIC2 OR LIC3"), 'pre-', '(LIC2 OR LIC3) AND pre-LIC1'),
- 'dual multi license' => array(array("LIC1","LIC2 OR LIC3", "Dual-license"), '', '(LIC2 OR LIC3) OR LIC1'),
- 'dual multi license with prefix' => array(array("LIC1","LIC2 OR LIC3", "Dual-license"), 'pre-', '(LIC2 OR LIC3) OR pre-LIC1'),
+ 'null' => array(null, $constTrue, ''),
+ 'empty array' => array(array(), $constTrue, ''),
+ 'empty array but prefix' => array(array(), $constFalse, ''),
+ 'single license' => array(array("LIC1"), $constTrue, 'LIC1'),
+ 'multiple licenses' => array(array("LIC1","LIC2","LIC3"), $constTrue, 'LIC1 AND LIC2 AND LIC3'),
+ 'dual license 1st pos' => array(array("Dual-license", "LIC2", "LIC3"), $constTrue, 'LIC2 OR LIC3'),
+ 'dual license 2nd pos' => array(array("LIC1", "Dual-license", "LIC3"), $constTrue, 'LIC1 OR LIC3'),
+ 'dual license 3rd pos' => array(array("LIC1", "LIC2", "Dual-license"), $constTrue, 'LIC1 OR LIC2'),
+ 'dual license with prefix' => array(array("LIC1","LIC2", "Dual-license"), function ($licId) { return $licId === 'LIC2';}, SpdxTwoUtils::$prefix.'LIC1 OR LIC2'),
+ 'dual license with prefix' => array(array("LIC1","LIC2", "Dual-license"), function ($licId) { return $licId === 'LIC1';}, 'LIC1 OR '.SpdxTwoUtils::$prefix.'LIC2'),
+ 'dual license with prefix ($constFalse)' => array(array("LIC1","LIC2", "Dual-license"), $constFalse, SpdxTwoUtils::$prefix.'LIC1 OR '.SpdxTwoUtils::$prefix.'LIC2'),
+ 'multiple dualLicense' => array(array("LIC1","LIC2 OR LIC3"), $constTrue, '(LIC2 OR LIC3) AND LIC1'),
+ 'multiple dualLicense with prefix' => array(array("LIC1","LIC2 OR LIC3"), $constFalse, '(LIC2 OR LIC3) AND '.SpdxTwoUtils::$prefix.'LIC1'),
+ 'dual multi license' => array(array("LIC1","LIC2 OR LIC3", "Dual-license"), $constTrue, '(LIC2 OR LIC3) OR LIC1'),
+ 'dual multi license with prefix' => array(array("LIC1","LIC2 OR LIC3", "Dual-license"), $constFalse, '(LIC2 OR LIC3) OR '.SpdxTwoUtils::$prefix.'LIC1'),
);
}
/**
* @dataProvider provideLicenseSet
* @param array $lics
* @param string $prefix
* @param string $expected
*/
public function testImplodeLicenses($lics, $prefix, $expected)
{
assertThat(SpdxTwoUtils::implodeLicenses($lics, $prefix), equalTo($expected));
}
}