1package db 2 3import ( 4 "fmt" 5 "io/ioutil" 6 "os" 7 8 "github.com/cheggaaa/pb/v3" 9 "github.com/jinzhu/gorm" 10 "github.com/knqyf263/go-cpe/common" 11 "github.com/knqyf263/go-cpe/naming" 12 "github.com/kotakanbe/go-cve-dictionary/config" 13 c "github.com/kotakanbe/go-cve-dictionary/config" 14 log "github.com/kotakanbe/go-cve-dictionary/log" 15 "github.com/kotakanbe/go-cve-dictionary/models" 16 sqlite3 "github.com/mattn/go-sqlite3" 17 18 // Required MySQL. See http://jinzhu.me/gorm/database.html#connecting-to-a-database 19 _ "github.com/jinzhu/gorm/dialects/mysql" 20 _ "github.com/jinzhu/gorm/dialects/postgres" 21 22 // Required SQLite3. 23 _ "github.com/jinzhu/gorm/dialects/sqlite" 24) 25 26// Supported DB dialects. 27const ( 28 dialectSqlite3 = "sqlite3" 29 dialectMysql = "mysql" 30 dialectPostgreSQL = "postgres" 31) 32 33// RDBDriver is Driver for RDB 34type RDBDriver struct { 35 name string 36 conn *gorm.DB 37} 38 39// Name return db name 40func (r *RDBDriver) Name() string { 41 return r.name 42} 43 44// NewRDB return RDB driver 45func NewRDB(dbType, dbpath string, debugSQL bool) (driver *RDBDriver, locked bool, err error) { 46 driver = &RDBDriver{ 47 name: dbType, 48 } 49 50 log.Debugf("Opening DB (%s).", driver.Name()) 51 if locked, err = driver.OpenDB(dbType, dbpath, debugSQL); err != nil { 52 return nil, locked, err 53 } 54 55 log.Debugf("Migrating DB (%s).", driver.Name()) 56 if err := driver.MigrateDB(); err != nil { 57 return nil, false, err 58 } 59 return driver, false, nil 60} 61 62// OpenDB opens Database 63func (r *RDBDriver) OpenDB(dbType, dbPath string, debugSQL bool) (locked bool, err error) { 64 r.conn, err = gorm.Open(dbType, dbPath) 65 if err != nil { 66 if r.name == dialectSqlite3 { 67 switch err.(sqlite3.Error).Code { 68 case sqlite3.ErrLocked, sqlite3.ErrBusy: 69 return true, err 70 } 71 } 72 return false, fmt.Errorf("Failed to open DB. dbtype: %s, dbpath: %s, err: %s", 73 dbType, dbPath, err) 74 } 75 if err := r.conn.LogMode(debugSQL).Error; err != nil { 76 return false, err 77 } 78 r.conn.LogMode(debugSQL) 79 return false, nil 80} 81 82// MigrateDB migrates Database 83func (r *RDBDriver) MigrateDB() error { 84 if err := r.conn.AutoMigrate( 85 &models.FeedMeta{}, 86 &models.CveDetail{}, 87 &models.NvdJSON{}, 88 &models.Jvn{}, 89 &models.Reference{}, 90 &models.Cert{}, 91 &models.Cpe{}, 92 &models.EnvCpe{}, 93 &models.Cwe{}, 94 &models.Affect{}, 95 &models.Cvss3{}, 96 &models.Cvss2{}, 97 &models.Cvss2Extra{}, 98 &models.Description{}, 99 ).Error; err != nil { 100 return fmt.Errorf("Failed to migrate. err: %s", err) 101 } 102 103 errs := []error{} 104 105 // CveID 106 errs = append(errs, r.conn.Model(&models.CveDetail{}). 107 AddIndex("idx_cve_detail_cveid", "cve_id").Error) 108 errs = append(errs, r.conn.Model(&models.NvdJSON{}). 109 AddIndex("idx_nvd_jsons_cveid", "cve_id").Error) 110 errs = append(errs, r.conn.Model(&models.Jvn{}). 111 AddIndex("idx_jvns_cveid", "cve_id").Error) 112 113 // CveDetailID 114 errs = append(errs, r.conn.Model(&models.NvdJSON{}). 115 AddIndex("idx_nvd_jsons_cve_detail_id", "cve_detail_id").Error) 116 errs = append(errs, r.conn.Model(&models.Jvn{}). 117 AddIndex("idx_jvns_cve_detail_id", "cve_detail_id").Error) 118 119 // references 120 errs = append(errs, r.conn.Model(&models.Reference{}). 121 AddIndex("idx_references_jvn_id", "jvn_id").Error) 122 errs = append(errs, r.conn.Model(&models.Reference{}). 123 AddIndex("idx_references_nvd_json_id", "nvd_json_id").Error) 124 125 // certs 126 errs = append(errs, r.conn.Model(&models.Cert{}). 127 AddIndex("idx_certs_jvn_id", "jvn_id").Error) 128 errs = append(errs, r.conn.Model(&models.Cert{}). 129 AddIndex("idx_certs_nvd_json_id", "nvd_json_id").Error) 130 131 // cpes 132 errs = append(errs, r.conn.Model(&models.Cpe{}). 133 AddIndex("idx_cpes_jvn_id", "jvn_id").Error) 134 errs = append(errs, r.conn.Model(&models.Cpe{}). 135 AddIndex("idx_cpes_nvd_json_id", "nvd_json_id").Error) 136 errs = append(errs, r.conn.Model(&models.Cpe{}). 137 AddIndex("idx_cpes_uri", "uri").Error) 138 errs = append(errs, r.conn.Model(&models.Cpe{}). 139 AddIndex("idx_cpes_formatted_string", "formatted_string").Error) 140 errs = append(errs, r.conn.Model(&models.Cpe{}). 141 AddIndex("idx_cpes_part", "part").Error) 142 errs = append(errs, r.conn.Model(&models.Cpe{}). 143 AddIndex("idx_cpes_vendor", "vendor").Error) 144 errs = append(errs, r.conn.Model(&models.Cpe{}). 145 AddIndex("idx_cpes_product", "product").Error) 146 147 // envcpes 148 errs = append(errs, r.conn.Model(&models.EnvCpe{}). 149 AddIndex("idx_envcpes_cpe_id", "cpe_id").Error) 150 errs = append(errs, r.conn.Model(&models.EnvCpe{}). 151 AddIndex("idx_envcpes_uri", "uri").Error) 152 errs = append(errs, r.conn.Model(&models.EnvCpe{}). 153 AddIndex("idx_envcpes_formatted_string", "formatted_string").Error) 154 155 // Cwes 156 errs = append(errs, r.conn.Model(&models.Cwe{}). 157 AddIndex("idx_cwes_jvn_id", "jvn_id").Error) 158 errs = append(errs, r.conn.Model(&models.Cwe{}). 159 AddIndex("idx_cwes_nvd_json_id", "nvd_json_id").Error) 160 161 // Affects 162 //TODO add index to vendor, product if needed 163 errs = append(errs, r.conn.Model(&models.Affect{}). 164 AddIndex("idx_affects_nvd_json_id", "nvd_json_id").Error) 165 166 // Cvss3 167 errs = append(errs, r.conn.Model(&models.Cvss3{}). 168 AddIndex("idx_cvss3_nvd_json_id", "nvd_json_id").Error) 169 errs = append(errs, r.conn.Model(&models.Cvss3{}). 170 AddIndex("idx_cvss3_jvn_id", "jvn_id").Error) 171 172 // Cvss2 173 errs = append(errs, r.conn.Model(&models.Cvss2{}). 174 AddIndex("idx_cvss2_jvn_id", "jvn_id").Error) 175 176 // Cvss2Extra 177 errs = append(errs, r.conn.Model(&models.Cvss2Extra{}). 178 AddIndex("idx_cvsss2_extra_nvd_json_id", "nvd_json_id").Error) 179 180 // Description 181 errs = append(errs, r.conn.Model(&models.Description{}). 182 AddIndex("idx_descriptions_nvd_json_id", "nvd_json_id").Error) 183 184 for _, e := range errs { 185 if e != nil { 186 return fmt.Errorf("Failed to create index. err: %s", e) 187 } 188 } 189 190 return nil 191} 192 193// Get Select Cve information from DB. 194func (r *RDBDriver) Get(cveID string) (*models.CveDetail, error) { 195 c := models.CveDetail{} 196 err := r.conn.Where(&models.CveDetail{CveID: cveID}).First(&c).Error 197 if err != nil && err != gorm.ErrRecordNotFound { 198 return nil, err 199 } 200 201 if c.ID == 0 { 202 return &models.CveDetail{}, nil 203 } 204 205 // JVN 206 jvn := models.Jvn{} 207 err = r.conn.Model(&c).Related(&jvn, "Jvn").Error 208 if err != nil && err != gorm.ErrRecordNotFound { 209 return nil, err 210 } 211 if jvn.ID == 0 { 212 c.Jvn = nil 213 } else { 214 c.Jvn = &jvn 215 } 216 if jvn.CveDetailID != 0 && jvn.ID != 0 { 217 jvnRefs := []models.Reference{} 218 err = r.conn.Model(&jvn).Related(&jvnRefs, "References").Error 219 if err != nil && err != gorm.ErrRecordNotFound { 220 return nil, err 221 } 222 c.Jvn.References = jvnRefs 223 224 certs := []models.Cert{} 225 err = r.conn.Model(&jvn).Related(&certs, "Certs").Error 226 if err != nil && err != gorm.ErrRecordNotFound { 227 return nil, err 228 } 229 c.Jvn.Certs = certs 230 231 cvss2 := models.Cvss2{} 232 err = r.conn.Model(&jvn).Related(&cvss2, "Cvss2").Error 233 if err != nil && err != gorm.ErrRecordNotFound { 234 return nil, err 235 } 236 c.Jvn.Cvss2 = cvss2 237 238 cvss3 := models.Cvss3{} 239 err = r.conn.Model(&jvn).Related(&cvss3, "Cvss3").Error 240 if err != nil && err != gorm.ErrRecordNotFound { 241 return nil, err 242 } 243 c.Jvn.Cvss3 = cvss3 244 245 // TODO commentout because JSON response size will be big. so Uncomment if needed. 246 // jvnCpes := []models.Cpe{} 247 // conn.Model(&jvn).Related(&jvnCpes, "Cpes") 248 // c.Jvn.Cpes = jvnCpes 249 } 250 251 // NVD JSON 252 json := models.NvdJSON{} 253 err = r.conn.Model(&c).Related(&json, "NvdJSON").Error 254 if err != nil && err != gorm.ErrRecordNotFound { 255 return nil, err 256 } 257 if json.ID == 0 { 258 c.NvdJSON = nil 259 } else { 260 c.NvdJSON = &json 261 } 262 if json.CveDetailID != 0 && json.ID != 0 { 263 descs := []models.Description{} 264 err = r.conn.Model(&json).Related(&descs, "Descriptions").Error 265 if err != nil && err != gorm.ErrRecordNotFound { 266 return nil, err 267 } 268 c.NvdJSON.Descriptions = descs 269 270 cvss2 := models.Cvss2Extra{} 271 err = r.conn.Model(&json).Related(&cvss2, "Cvss2").Error 272 if err != nil && err != gorm.ErrRecordNotFound { 273 return nil, err 274 } 275 c.NvdJSON.Cvss2 = cvss2 276 277 cvss3 := models.Cvss3{} 278 err = r.conn.Model(&json).Related(&cvss3, "Cvss3").Error 279 if err != nil && err != gorm.ErrRecordNotFound { 280 return nil, err 281 } 282 c.NvdJSON.Cvss3 = cvss3 283 284 cwes := []models.Cwe{} 285 err = r.conn.Model(&json).Related(&cwes, "Cwes").Error 286 if err != nil && err != gorm.ErrRecordNotFound { 287 return nil, err 288 } 289 c.NvdJSON.Cwes = cwes 290 291 affects := []models.Affect{} 292 err = r.conn.Model(&json).Related(&affects, "Affects").Error 293 if err != nil && err != gorm.ErrRecordNotFound { 294 return nil, err 295 } 296 c.NvdJSON.Affects = affects 297 298 refs := []models.Reference{} 299 err = r.conn.Model(&json).Related(&refs, "References").Error 300 if err != nil && err != gorm.ErrRecordNotFound { 301 return nil, err 302 } 303 c.NvdJSON.References = refs 304 305 certs := []models.Cert{} 306 err = r.conn.Model(&json).Related(&certs, "Certs").Error 307 if err != nil && err != gorm.ErrRecordNotFound { 308 return nil, err 309 } 310 c.NvdJSON.Certs = certs 311 312 cpes := []models.Cpe{} 313 err = r.conn.Model(&json).Related(&cpes, "Cpes").Error 314 if err != nil && err != gorm.ErrRecordNotFound { 315 return nil, err 316 } 317 for i, cpe := range cpes { 318 envCpes := []models.EnvCpe{} 319 err = r.conn.Model(&cpe).Related(&envCpes, "EnvCpes").Error 320 if err != nil && err != gorm.ErrRecordNotFound { 321 return nil, err 322 } 323 cpes[i].EnvCpes = envCpes 324 } 325 c.NvdJSON.Cpes = cpes 326 } 327 328 return &c, nil 329} 330 331// GetMulti Select Cves information from DB. 332func (r *RDBDriver) GetMulti(cveIDs []string) (map[string]models.CveDetail, error) { 333 cveDetails := map[string]models.CveDetail{} 334 for _, cveID := range cveIDs { 335 cve, err := r.Get(cveID) 336 if err != nil { 337 return nil, err 338 } 339 cveDetails[cveID] = *cve 340 } 341 return cveDetails, nil 342} 343 344// CloseDB close Database 345func (r *RDBDriver) CloseDB() (err error) { 346 if err = r.conn.Close(); err != nil { 347 log.Errorf("Failed to close DB. Type: %s. err: %s", r.name, err) 348 return 349 } 350 return 351} 352 353// getMatchingCpes returns matching CPEs by pseudo-CPE 354func (r *RDBDriver) getMatchingCpes(uri string) ([]models.Cpe, error) { 355 356 // parse wfn, get vendor, product 357 specified, err := naming.UnbindURI(uri) 358 if err != nil { 359 return nil, err 360 } 361 // select from cpe by vendor, product 362 cpes := []models.Cpe{} 363 err = r.conn.Where(&models.Cpe{ 364 CpeBase: models.CpeBase{ 365 CpeWFN: models.CpeWFN{ 366 Vendor: fmt.Sprintf("%s", specified.Get(common.AttributeVendor)), 367 Product: fmt.Sprintf("%s", specified.Get(common.AttributeProduct)), 368 }, 369 }}).Find(&cpes).Error 370 if err != nil && err != gorm.ErrRecordNotFound { 371 return nil, err 372 } 373 374 log.Debugf("specified: %s", uri) 375 filtered := []models.Cpe{} 376 checkedIDs := map[uint]bool{} 377 for _, cpe := range cpes { 378 match, err := match(uri, cpe) 379 if err != nil { 380 log.Debugf("Failed to compare the version:%s %s cpe_id:%d %#v", 381 err, uri, cpe.ID, cpe) 382 383 // Try to exact match by vendor, product and version if the version in CPE is not a semVer style. 384 if cpe.NvdJSONID != 0 { 385 if _, ok := checkedIDs[cpe.NvdJSONID]; ok { 386 continue 387 } 388 affects := []models.Affect{} 389 result := r.conn.Where(&models.Affect{NvdJSONID: cpe.NvdJSONID}).Find(&affects) 390 if result.Error != nil && result.Error != gorm.ErrRecordNotFound { 391 return nil, result.Error 392 } 393 394 ok, err := matchExactByAffects(uri, affects) 395 if err != nil { 396 return nil, err 397 } 398 if ok { 399 filtered = append(filtered, cpe) 400 } 401 checkedIDs[cpe.NvdJSONID] = true 402 } 403 } else if match { 404 filtered = append(filtered, cpe) 405 } 406 } 407 408 return filtered, nil 409} 410 411// GetCveIDsByCpeURI Select Cve Ids by by pseudo-CPE 412func (r *RDBDriver) GetCveIDsByCpeURI(uri string) ([]string, error) { 413 filtered, err := r.getMatchingCpes(uri) 414 if err != nil { 415 return nil, err 416 } 417 418 cveIDs := make([]string, len(filtered)) 419 420 // The cpes table is de-normalized. So the `First` is correct, don't be mislead. 421 for i, cpe := range filtered { 422 if cpe.JvnID != 0 { 423 jvn := models.Jvn{} 424 err = r.conn.Select("cve_id").Where("ID = ?", cpe.JvnID).First(&jvn).Error 425 if err != nil && err != gorm.ErrRecordNotFound { 426 return nil, err 427 } 428 cveIDs[i] = jvn.CveID 429 } else if cpe.NvdJSONID != 0 { 430 json := models.NvdJSON{} 431 err = r.conn.Select("cve_id").Where("ID = ?", cpe.NvdJSONID).First(&json).Error 432 if err != nil && err != gorm.ErrRecordNotFound { 433 return nil, err 434 } 435 cveIDs[i] = json.CveID 436 } 437 438 // If we didn't find a CVE something is weird. 439 if cveIDs[i] == "" { 440 log.Infof("Missing cve_id for %s (id: %d)", cpe.URI, cpe.ID) 441 } 442 } 443 444 return cveIDs, nil 445} 446 447// GetByCpeURI Select Cve information from DB. 448func (r *RDBDriver) GetByCpeURI(uri string) ([]models.CveDetail, error) { 449 cveIDs, err := r.GetCveIDsByCpeURI(uri) 450 if err != nil { 451 return nil, err 452 } 453 454 cveDetails, err := r.GetMulti(cveIDs) 455 if err != nil { 456 return nil, err 457 } 458 459 // Convert from map to array 460 details := make([]models.CveDetail, len(cveDetails)) 461 i := 0 462 for _, d := range cveDetails { 463 details[i] = d 464 i++ 465 log.Debugf("%s", d.CveID) 466 } 467 return details, nil 468} 469 470// InsertJvn inserts Cve Information into DB 471func (r *RDBDriver) InsertJvn(cves []models.CveDetail) error { 472 log.Infof("Inserting fetched CVEs...") 473 bar := pb.New(len(cves)) 474 if c.Conf.Quiet { 475 bar.SetWriter(ioutil.Discard) 476 } else { 477 bar.SetWriter(os.Stderr) 478 } 479 bar.Start() 480 481 var refreshedJvns []string 482 for chunked := range chunkSlice(cves, 10) { 483 tx := r.conn.Begin() 484 if tx.Error != nil { 485 return tx.Error 486 } 487 for _, c := range chunked { 488 bar.Increment() 489 490 // select old record. 491 old := models.CveDetail{} 492 result := tx.Where(&models.CveDetail{CveID: c.CveID}).First(&old) 493 if result.Error != nil && result.Error != gorm.ErrRecordNotFound { 494 return rollback(tx, result.Error) 495 } 496 if result.RecordNotFound() || old.ID == 0 { 497 if err := tx.Create(&c).Error; err != nil { 498 return rollback(tx, err) 499 } 500 refreshedJvns = append(refreshedJvns, c.CveID) 501 continue 502 } 503 504 if !result.RecordNotFound() { 505 // select Jvn from db 506 jvn := models.Jvn{} 507 err := tx.Model(&old).Related(&jvn, "Jvn").Error 508 if err != nil && err != gorm.ErrRecordNotFound { 509 return rollback(tx, err) 510 } 511 512 if jvn.CveDetailID == 0 { 513 c.Jvn.CveDetailID = old.ID 514 if err := tx.Create(&c.Jvn).Error; err != nil { 515 return rollback(tx, err) 516 } 517 refreshedJvns = append(refreshedJvns, c.CveID) 518 continue 519 } 520 521 // Refresh JVN Record. 522 523 if !config.Conf.Force { 524 if jvn.LastModifiedDate.Equal(c.Jvn.LastModifiedDate) || 525 jvn.LastModifiedDate.After(c.Jvn.LastModifiedDate) { 526 // skip if the record has already been in DB and not modified. 527 // log.Debugf("Not modified. old: %s", old.CveID) 528 continue 529 } 530 } 531 log.Debugf("Newer record found. CveID: %s, old: %s, new: %s", 532 c.CveID, jvn.LastModifiedDate, c.Jvn.LastModifiedDate) 533 534 // Delete old References 535 refs := []models.Reference{} 536 err = tx.Model(&jvn).Related(&refs, "References").Error 537 if err != nil && err != gorm.ErrRecordNotFound { 538 return rollback(tx, err) 539 } 540 for _, r := range refs { 541 if err := tx.Unscoped().Delete(r).Error; err != nil { 542 return rollback(tx, err) 543 } 544 } 545 546 // Delete Certs 547 certs := []models.Cert{} 548 err = tx.Model(&jvn).Related(&certs, "Certs").Error 549 if err != nil && err != gorm.ErrRecordNotFound { 550 return rollback(tx, err) 551 } 552 for _, l := range certs { 553 if err := tx.Unscoped().Delete(l).Error; err != nil { 554 return rollback(tx, err) 555 } 556 } 557 558 // Delete Cvss2 559 cvss2 := []models.Cvss2{} 560 err = tx.Model(&jvn).Related(&cvss2, "Cvss2").Error 561 if err != nil && err != gorm.ErrRecordNotFound { 562 return rollback(tx, err) 563 } 564 for _, cvss := range cvss2 { 565 if err := tx.Unscoped().Delete(cvss).Error; err != nil { 566 return rollback(tx, err) 567 } 568 } 569 570 // Delete Cvss3 571 cvss3 := []models.Cvss3{} 572 err = tx.Model(&jvn).Related(&cvss3, "Cvss3").Error 573 if err != nil && err != gorm.ErrRecordNotFound { 574 return rollback(tx, err) 575 } 576 for _, cvss := range cvss3 { 577 if err := tx.Unscoped().Delete(cvss).Error; err != nil { 578 return rollback(tx, err) 579 } 580 } 581 582 // Delete old Cpes 583 cpes := []models.Cpe{} 584 err = tx.Model(&jvn).Related(&cpes, "Cpes").Error 585 if err != nil && err != gorm.ErrRecordNotFound { 586 return rollback(tx, err) 587 } 588 for _, cpe := range cpes { 589 if err := tx.Unscoped().Delete(cpe).Error; err != nil { 590 return rollback(tx, err) 591 } 592 } 593 594 // Delete old Jvn 595 if err := tx.Unscoped().Delete(&jvn).Error; err != nil { 596 return rollback(tx, err) 597 } 598 599 // Insert Jvn 600 c.Jvn.CveDetailID = old.ID 601 if err := tx.Create(&c.Jvn).Error; err != nil { 602 return rollback(tx, err) 603 } 604 refreshedJvns = append(refreshedJvns, c.CveID) 605 } 606 } 607 if err := tx.Commit().Error; err != nil { 608 return rollback(tx, err) 609 } 610 } 611 bar.Finish() 612 log.Infof("Refreshed %d Jvns.", len(refreshedJvns)) 613 log.Debugf("%v", refreshedJvns) 614 return nil 615} 616 617func rollback(tx *gorm.DB, err error) error { 618 if err := tx.Rollback().Error; err != nil { 619 return err 620 } 621 return err 622} 623 624// CountNvd count nvd table 625func (r *RDBDriver) CountNvd() (int, error) { 626 var count int 627 err := r.conn.Model(&models.NvdJSON{}).Count(&count).Error 628 if err != nil && err != gorm.ErrRecordNotFound { 629 return 0, err 630 } 631 return count, nil 632} 633 634// InsertNvdJSON Cve information from DB. 635func (r *RDBDriver) InsertNvdJSON(cves []models.CveDetail) (err error) { 636 log.Infof("Inserting fetched CVEs...") 637 bar := pb.New(len(cves)) 638 if c.Conf.Quiet { 639 bar.SetWriter(ioutil.Discard) 640 } else { 641 bar.SetWriter(os.Stderr) 642 } 643 bar.Start() 644 645 var refreshedNvds []string 646 for chunked := range chunkSlice(cves, 10) { 647 tx := r.conn.Begin() 648 if tx.Error != nil { 649 return tx.Error 650 } 651 for _, c := range chunked { 652 bar.Increment() 653 654 // select old record. 655 old := models.CveDetail{} 656 result := tx.Where(&models.CveDetail{CveID: c.CveID}).First(&old) 657 if result.Error != nil && result.Error != gorm.ErrRecordNotFound { 658 return rollback(tx, result.Error) 659 } 660 if result.RecordNotFound() || old.ID == 0 { 661 // log.Debugf("CveDetail is not found: %s", c.CveID) 662 if err := tx.Create(&c).Error; err != nil { 663 return rollback(tx, err) 664 } 665 refreshedNvds = append(refreshedNvds, c.CveID) 666 continue 667 } 668 669 if !result.RecordNotFound() { 670 // select NvdJSON from db 671 json := models.NvdJSON{} 672 err = tx.Model(&old).Related(&json, "NvdJSON").Error 673 if err != nil && err != gorm.ErrRecordNotFound { 674 return rollback(tx, err) 675 } 676 677 // If the NVD JSON hasn't been inserted yet, insert the NVD JSON newly and associate it with existing CveDetail. 678 if json.CveDetailID == 0 { 679 log.Debugf("CveDetail found but no NVD JSON: %s", c.CveID) 680 c.NvdJSON.CveDetailID = old.ID 681 if err := tx.Create(&c.NvdJSON).Error; err != nil { 682 return rollback(tx, err) 683 } 684 refreshedNvds = append(refreshedNvds, c.CveID) 685 continue 686 } 687 688 // If the NVD JSON has already been inserted, Refresh to new NVD JSON Record. 689 690 if !config.Conf.Force { 691 if json.LastModifiedDate.Equal(c.NvdJSON.LastModifiedDate) || 692 json.LastModifiedDate.After(c.NvdJSON.LastModifiedDate) { 693 // skip if the record has already been in DB and not modified. 694 log.Debugf("Not modified. old: %s", old.CveID) 695 continue 696 } 697 } 698 699 log.Debugf("newer Record found. CveID: %s, old: %s, new: %s", 700 c.CveID, json.LastModifiedDate, c.NvdJSON.LastModifiedDate) 701 702 // Delete old Descriptions 703 descs := []models.Description{} 704 err = tx.Model(&json).Related(&descs, "Descriptions").Error 705 if err != nil && err != gorm.ErrRecordNotFound { 706 return rollback(tx, err) 707 } 708 for _, desc := range descs { 709 if err := tx.Unscoped().Delete(desc).Error; err != nil { 710 return rollback(tx, err) 711 } 712 } 713 714 // Delete Cvss2 715 cvss2 := []models.Cvss2Extra{} 716 err = tx.Model(&json).Related(&cvss2, "Cvss2").Error 717 if err != nil && err != gorm.ErrRecordNotFound { 718 return rollback(tx, err) 719 } 720 for _, cvss := range cvss2 { 721 if err := tx.Unscoped().Delete(cvss).Error; err != nil { 722 return rollback(tx, err) 723 } 724 } 725 726 // Delete Cvss3 727 cvss3 := []models.Cvss3{} 728 err = tx.Model(&json).Related(&cvss3, "Cvss3").Error 729 if err != nil && err != gorm.ErrRecordNotFound { 730 return rollback(tx, err) 731 } 732 for _, cvss := range cvss3 { 733 if err := tx.Unscoped().Delete(cvss).Error; err != nil { 734 return rollback(tx, err) 735 } 736 } 737 738 // Delete old CWE 739 cwes := []models.Cwe{} 740 err = tx.Model(&json).Related(&cwes, "Cwes").Error 741 if err != nil && err != gorm.ErrRecordNotFound { 742 return rollback(tx, err) 743 } 744 for _, cwe := range cwes { 745 if err := tx.Unscoped().Delete(cwe).Error; err != nil { 746 return rollback(tx, err) 747 } 748 } 749 750 // Delete old Cpes and EnvEpes 751 cpes := []models.Cpe{} 752 err = tx.Model(&json).Related(&cpes, "Cpes").Error 753 if err != nil && err != gorm.ErrRecordNotFound { 754 return rollback(tx, err) 755 } 756 for _, cpe := range cpes { 757 envs := []models.EnvCpe{} 758 err = tx.Model(&cpe).Related(&envs, "EnvCpes").Error 759 if err != nil && err != gorm.ErrRecordNotFound { 760 return rollback(tx, err) 761 } 762 for _, env := range envs { 763 if err := tx.Unscoped().Delete(env).Error; err != nil { 764 return rollback(tx, err) 765 } 766 } 767 768 if err := tx.Unscoped().Delete(cpe).Error; err != nil { 769 return rollback(tx, err) 770 } 771 } 772 773 // Delete old Affects 774 affects := []models.Affect{} 775 err = tx.Model(&json).Related(&affects, "Affects").Error 776 if err != nil && err != gorm.ErrRecordNotFound { 777 return rollback(tx, err) 778 } 779 for _, r := range affects { 780 if err := tx.Unscoped().Delete(r).Error; err != nil { 781 return rollback(tx, err) 782 } 783 } 784 785 // Delete old References 786 refs := []models.Reference{} 787 err = tx.Model(&json).Related(&refs, "References").Error 788 if err != nil && err != gorm.ErrRecordNotFound { 789 return rollback(tx, err) 790 } 791 for _, r := range refs { 792 if err := tx.Unscoped().Delete(r).Error; err != nil { 793 return rollback(tx, err) 794 } 795 } 796 797 // Delete Certs 798 certs := []models.Cert{} 799 err = tx.Model(&json).Related(&certs, "Certs").Error 800 if err != nil && err != gorm.ErrRecordNotFound { 801 return rollback(tx, err) 802 } 803 for _, l := range certs { 804 if err := tx.Unscoped().Delete(l).Error; err != nil { 805 return rollback(tx, err) 806 } 807 } 808 809 // Delete old NvdJSON 810 if err := tx.Unscoped().Delete(&json).Error; err != nil { 811 return rollback(tx, err) 812 } 813 814 // Insert Nvd JSON 815 c.NvdJSON.CveDetailID = old.ID 816 if err := tx.Create(&c.NvdJSON).Error; err != nil { 817 return rollback(tx, err) 818 } 819 refreshedNvds = append(refreshedNvds, c.CveID) 820 } 821 } 822 if err := tx.Commit().Error; err != nil { 823 return rollback(tx, err) 824 } 825 } 826 bar.Finish() 827 828 log.Infof("Refreshed %d NvdJSON.", len(refreshedNvds)) 829 // log.Debugf("%v", refreshedNvds) 830 return nil 831} 832 833// GetFetchedFeedMeta selects fetchmeta of the year 834func (r *RDBDriver) GetFetchedFeedMeta(url string) (*models.FeedMeta, error) { 835 meta := models.FeedMeta{} 836 m := &models.FeedMeta{ 837 URL: url, 838 } 839 err := r.conn.Where(m).First(&meta).Error 840 if err != nil && err != gorm.ErrRecordNotFound { 841 return nil, err 842 } 843 return &meta, nil 844} 845 846// UpsertFeedHash selects sha1 in metafile of the year 847func (r *RDBDriver) UpsertFeedHash(mm models.FeedMeta) error { 848 meta := models.FeedMeta{} 849 m := &models.FeedMeta{ 850 URL: mm.URL, 851 } 852 err := r.conn.Where(m).First(&meta).Error 853 if err != nil && err != gorm.ErrRecordNotFound { 854 return err 855 } 856 857 tx := r.conn.Begin() 858 if tx.Error != nil { 859 return tx.Error 860 } 861 862 if err == gorm.ErrRecordNotFound { 863 m.Hash = mm.Hash 864 m.LastModifiedDate = mm.LastModifiedDate 865 if err := tx.Create(m).Error; err != nil { 866 return rollback(tx, err) 867 } 868 } else { 869 meta.Hash = mm.Hash 870 meta.LastModifiedDate = mm.LastModifiedDate 871 if err := tx.Save(&meta).Error; err != nil { 872 return rollback(tx, err) 873 } 874 } 875 876 if err := tx.Commit().Error; err != nil { 877 return rollback(tx, err) 878 } 879 return nil 880} 881 882// GetFetchedFeedMetas selects a list of FeedMeta 883func (r *RDBDriver) GetFetchedFeedMetas() ([]models.FeedMeta, error) { 884 metas := []models.FeedMeta{} 885 err := r.conn.Find(&metas).Error 886 if err != nil && err != gorm.ErrRecordNotFound { 887 return nil, err 888 } 889 return metas, nil 890} 891