Skip to content

ROX-12975: Extend Enricher to accept NodeInventory as a second parameter #4492

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vikin91 merged 19 commits into from
Feb 2, 2023
Merged

ROX-12975: Extend Enricher to accept NodeInventory as a second parameter #4492

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
fb576e4
Extend Enricher to accept NodeInventory as a second parameter
vikin91 Jan 17, 2023
00af4e5
Add imageScanServiceClient to clairify node scanner
vikin91 Jan 20, 2023
39573b3
Fixes and tmp changes:
vikin91 Jan 17, 2023
b58390f
Bump scanner version and dependency
vikin91 Jan 23, 2023
fda68b9
Update Scanner dep from master
vikin91 Jan 24, 2023
0aadc6d
Add comments explaining the data race between the pipelines
vikin91 Jan 24, 2023
f869fca
Rerun code generation
vikin91 Jan 24, 2023
acc35b2
Add more test coverage for convertNodeToVulnRequest
vikin91 Jan 26, 2023
b938957
Refactor & comment on nil-ness of EnrichNodeWithInventory params
vikin91 Jan 26, 2023
a6cafdd
Fix typo
vikin91 Jan 26, 2023
562e75e
Rename import to riskManager
vikin91 Jan 26, 2023
baa2b6e
Unexport nodeinventory.newPipeline
vikin91 Jan 26, 2023
f5f5659
Add informational debug logging
vikin91 Jan 26, 2023
ca9a113
Tmp: Pull scanner code from Victor's branch to check content sets
vikin91 Jan 27, 2023
06dab9c
Ignore requests to delete node inventories
vikin91 Jan 27, 2023
dc23382
Use scanner version and deps from master
vikin91 Jan 30, 2023
31e6b7d
Run go mod tidy again
vikin91 Jan 30, 2023
4e20215
Minor: Correct log message
vikin91 Feb 1, 2023
1684b59
Take scanner deps from master
vikin91 Feb 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions central/imageintegration/service/service_impl_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,10 @@ func (*fakeNodeScanner) GetNodeScan(*storage.Node) (*storage.NodeScan, error) {
panic("implement me")
}

func (*fakeNodeScanner) GetNodeInventoryScan(node *storage.Node, inv *storage.NodeInventory) (*storage.NodeScan, error) {
panic("implement me")
}

func (*fakeNodeScanner) TestNodeScanner() error {
return nil
}
Expand Down
64 changes: 56 additions & 8 deletions central/sensor/service/pipeline/nodeinventory/pipeline.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,18 @@ import (
"context"

"github.com/pkg/errors"
clusterDataStore "github.com/stackrox/rox/central/cluster/datastore"
"github.com/stackrox/rox/central/enrichment"
countMetrics "github.com/stackrox/rox/central/metrics"
nodeDatastore "github.com/stackrox/rox/central/node/datastore"
riskManager "github.com/stackrox/rox/central/risk/manager"
"github.com/stackrox/rox/central/sensor/service/common"
"github.com/stackrox/rox/central/sensor/service/pipeline"
"github.com/stackrox/rox/central/sensor/service/pipeline/reconciliation"
"github.com/stackrox/rox/generated/internalapi/central"
"github.com/stackrox/rox/pkg/logging"
"github.com/stackrox/rox/pkg/metrics"
"github.com/stackrox/rox/pkg/nodes/enricher"
)

var (
Expand All @@ -19,15 +24,24 @@ var (

// GetPipeline returns an instantiation of this particular pipeline
func GetPipeline() pipeline.Fragment {
return NewPipeline()
return newPipeline(clusterDataStore.Singleton(), nodeDatastore.Singleton(), enrichment.NodeEnricherSingleton(), riskManager.Singleton())
}

// NewPipeline returns a new instance of Pipeline.
func NewPipeline() pipeline.Fragment {
return &pipelineImpl{}
// newPipeline returns a new instance of Pipeline.
func newPipeline(clusters clusterDataStore.DataStore, nodes nodeDatastore.DataStore, enricher enricher.NodeEnricher, riskManager riskManager.Manager) pipeline.Fragment {
return &pipelineImpl{
clusterStore: clusters,
nodeDatastore: nodes,
enricher: enricher,
riskManager: riskManager,
}
}

type pipelineImpl struct {
clusterStore clusterDataStore.DataStore
nodeDatastore nodeDatastore.DataStore
enricher enricher.NodeEnricher
riskManager riskManager.Manager
}

func (p *pipelineImpl) Reconcile(ctx context.Context, clusterID string, storeMap *reconciliation.StoreMap) error {
Expand All @@ -43,14 +57,48 @@ func (p *pipelineImpl) Run(ctx context.Context, clusterID string, msg *central.M
defer countMetrics.IncrementResourceProcessedCounter(pipeline.ActionToOperation(msg.GetEvent().GetAction()), metrics.NodeInventory)

event := msg.GetEvent()
nodeInventory := event.GetNodeInventory()
if nodeInventory == nil {
ninv := event.GetNodeInventory()
if ninv == nil {
return errors.Errorf("unexpected resource type %T for node inventory", event.GetResource())
}

// TODO(ROX-12975): Handle nodeInventory
log.Infof("Central received NodeInventory for Node name='%s' ID='%s'", nodeInventory.GetNodeName(), nodeInventory.GetNodeId())
log.Infof("Received NodeInventory for Node name='%s' ID='%s'", ninv.GetNodeName(), ninv.GetNodeId())
log.Debugf("NodeInventory for name='%s' contains %d packages to scan from %d content sets", ninv.GetNodeName(),
len(ninv.GetComponents().GetRhelComponents()), len(ninv.GetComponents().GetRhelContentSets()))

if event.GetAction() == central.ResourceAction_REMOVE_RESOURCE {
log.Warn("Deletion of NodeInventories is not supported")
return nil
}

ninv = ninv.Clone()

// TODO(ROX-14484): Resolve the race between pipelines - Start of critical section
node, found, err := p.nodeDatastore.GetNode(ctx, ninv.GetNodeId())
if err != nil || !found {
log.Warnf("Node ID %s not found when processing NodeInventory", ninv.GetNodeId())
return errors.WithMessagef(err, "processing node inventory for node '%s'", ninv.GetNodeId())
}
log.Debugf("Node ID %s found. Will enrich Node with NodeInventory", ninv.GetNodeId())

err = p.enricher.EnrichNodeWithInventory(node, ninv)
if err != nil {
log.Warnf("enriching node with node inventory %s:%s: %v", node.GetClusterName(), node.GetName(), err)
}

log.Debugf("NodeInventory for name='%s' has been scanned and contains %d results", ninv.GetNodeName(),
len(node.GetScan().GetComponents()))

// Here NodeInventory stops to matter. All data required for the DB and UI is in node.NodeScan already

if err := p.riskManager.CalculateRiskAndUpsertNode(node); err != nil {
err = errors.Wrapf(err, "upserting node %s:%s into datastore", node.GetClusterName(), node.GetName())
log.Error(err)
return err
}
// TODO(ROX-14484): Resolve the race between pipelines - End of critical section (when CalculateRiskAndUpsertNode finishes)
// We will loose data written in the node pipeline if the node pipeline writes an update to the DB
// while this pipeline is in the critical section!
return nil
}

Expand Down
46 changes: 23 additions & 23 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ go 1.18
require (
cloud.google.com/go/compute/metadata v0.2.3
cloud.google.com/go/containeranalysis v0.6.0
cloud.google.com/go/storage v1.28.1
cloud.google.com/go/storage v1.29.0
github.com/BurntSushi/toml v1.2.1
github.com/Masterminds/semver v1.5.0
github.com/Masterminds/sprig/v3 v3.2.3
Expand All @@ -18,14 +18,14 @@ require (
github.com/blevesearch/bleve v1.0.14
github.com/cenkalti/backoff/v3 v3.2.2
github.com/cloudflare/cfssl v1.6.3
github.com/containers/image/v5 v5.23.1
github.com/containers/image/v5 v5.24.0
github.com/coreos/go-oidc/v3 v3.5.0
github.com/coreos/go-systemd/v22 v22.5.0
github.com/dave/jennifer v1.6.0
github.com/dexidp/dex v0.0.0-20220607113954-3836196af2e7
github.com/docker/distribution v2.8.1+incompatible
// If this is updated, be sure to check the version of github.com/opencontainers/runc used.
github.com/docker/docker v20.10.21+incompatible
github.com/docker/docker v20.10.23+incompatible
github.com/docker/go-connections v0.4.0
github.com/docker/go-units v0.5.0
github.com/facebookincubator/nvdtools v0.1.5
Expand Down Expand Up @@ -86,7 +86,7 @@ require (
github.com/stackrox/external-network-pusher v0.0.0-20210419192707-074af92bbfa7
github.com/stackrox/helmtest v0.0.0-20220930104945-c4a3c15e834a
github.com/stackrox/k8s-istio-cve-pusher v0.0.0-20210422200002-d89f671ac4f5
github.com/stackrox/scanner v0.0.0-20230120022619-96107ead11f0
github.com/stackrox/scanner v0.0.0-20230131200500-e3d8a27d67b7
github.com/stretchr/testify v1.8.1
github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c
github.com/tidwall/gjson v1.14.1
Expand All @@ -96,7 +96,7 @@ require (
go.etcd.io/bbolt v1.3.6
go.uber.org/atomic v1.10.0
go.uber.org/goleak v1.2.0
go.uber.org/zap v1.23.0
go.uber.org/zap v1.24.0
golang.org/x/crypto v0.5.0
golang.org/x/exp v0.0.0-20220823124025-807a23277127
golang.org/x/net v0.5.0
Expand All @@ -107,8 +107,8 @@ require (
golang.org/x/tools v0.5.0
golang.stackrox.io/grpc-http1 v0.2.6
google.golang.org/api v0.107.0
google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef
google.golang.org/grpc v1.52.0
google.golang.org/genproto v0.0.0-20230131230820-1c016267d619
google.golang.org/grpc v1.52.3
google.golang.org/grpc/examples v0.0.0-20210902184326-c93e472777b9
gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5
gopkg.in/square/go-jose.v2 v2.6.0
Expand Down Expand Up @@ -140,7 +140,7 @@ require (
)

require (
cloud.google.com/go v0.105.0 // indirect
cloud.google.com/go v0.107.0 // indirect
cloud.google.com/go/compute v1.14.0 // indirect
cloud.google.com/go/iam v0.8.0 // indirect
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
Expand All @@ -155,7 +155,7 @@ require (
github.com/Masterminds/semver/v3 v3.2.0 // indirect
github.com/Masterminds/squirrel v1.5.3 // indirect
github.com/Microsoft/go-winio v0.6.0 // indirect
github.com/Microsoft/hcsshim v0.9.5 // indirect
github.com/Microsoft/hcsshim v0.9.6 // indirect
github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 // indirect
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
github.com/acomagu/bufpipe v1.0.3 // indirect
Expand All @@ -169,18 +169,18 @@ require (
github.com/blevesearch/go-porterstemmer v1.0.3 // indirect
github.com/blevesearch/mmap-go v1.0.2 // indirect
github.com/blevesearch/segment v0.9.0 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect
github.com/containerd/cgroups v1.0.4 // indirect
github.com/containerd/containerd v1.6.12 // indirect
github.com/containerd/continuity v0.3.0 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.12.1 // indirect
github.com/containers/storage v1.43.0 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.13.0 // indirect
github.com/containers/storage v1.45.3 // indirect
github.com/couchbase/ghistogram v0.1.0 // indirect
github.com/couchbase/moss v0.1.0 // indirect
github.com/couchbase/vellum v1.0.2 // indirect
github.com/creack/pty v1.1.18 // indirect
github.com/cyberphone/json-canonicalization v0.0.0-20210823021906-dc406ceaf94b // indirect
github.com/cyberphone/json-canonicalization v0.0.0-20220623050100-57a0ce2678a7 // indirect
github.com/cyphar/filepath-securejoin v0.2.3 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/docker/cli v20.10.20+incompatible // indirect
Expand Down Expand Up @@ -216,7 +216,7 @@ require (
github.com/go-openapi/validate v0.22.0 // indirect
github.com/go-playground/locales v0.14.0 // indirect
github.com/go-playground/universal-translator v0.18.0 // indirect
github.com/go-playground/validator/v10 v10.11.0 // indirect
github.com/go-playground/validator/v10 v10.11.1 // indirect
github.com/gobuffalo/flect v0.2.5 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/golang/glog v1.0.0 // indirect
Expand Down Expand Up @@ -257,8 +257,8 @@ require (
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/kevinburke/ssh_config v1.2.0 // indirect
github.com/klauspost/compress v1.15.14 // indirect
github.com/klauspost/pgzip v1.2.5 // indirect
github.com/klauspost/compress v1.15.15 // indirect
github.com/klauspost/pgzip v1.2.6-0.20220930104621-17e8dac29df8 // indirect
github.com/knqyf263/go-rpm-version v0.0.0-20220614171824-631e686d1075 // indirect
github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
Expand All @@ -268,7 +268,7 @@ require (
github.com/mattermost/xml-roundtrip-validator v0.1.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/mattn/go-runewidth v0.0.13 // indirect
github.com/mattn/go-runewidth v0.0.14 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mholt/archiver/v3 v3.5.1 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
Expand Down Expand Up @@ -298,8 +298,8 @@ require (
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
github.com/pierrec/lz4/v4 v4.1.2 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/prometheus/procfs v0.9.0 // indirect
github.com/rivo/uniseg v0.4.3 // indirect
github.com/rogpeppe/go-internal v1.8.1 // indirect
github.com/rubenv/sql-migrate v1.1.1 // indirect
github.com/russross/blackfriday v1.6.0 // indirect
Expand All @@ -309,7 +309,7 @@ require (
github.com/segmentio/backo-go v1.0.1 // indirect
github.com/shibumi/go-pathspec v1.3.0 // indirect
github.com/shopspring/decimal v1.2.0 // indirect
github.com/sigstore/rekor v0.12.1-0.20220915152154-4bb6f441c1b2 // indirect
github.com/sigstore/rekor v1.0.1 // indirect
github.com/sirupsen/logrus v1.9.0 // indirect
github.com/spf13/afero v1.8.2 // indirect
github.com/spf13/cast v1.5.0 // indirect
Expand All @@ -324,7 +324,7 @@ require (
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect
github.com/tent/canonical-json-go v0.0.0-20130607151641-96e4ba3a7613 // indirect
github.com/theupdateframework/go-tuf v0.5.2-0.20220930112810-3890c1e7ace4 // indirect
github.com/theupdateframework/go-tuf v0.5.2-0.20221207161717-9cb61d6e65f5 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 // indirect
Expand All @@ -342,10 +342,10 @@ require (
github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect
github.com/zmap/zcrypto v0.0.0-20220402174210-599ec18ecbac // indirect
github.com/zmap/zlint/v3 v3.4.0 // indirect
go.mongodb.org/mongo-driver v1.10.0 // indirect
go.mongodb.org/mongo-driver v1.11.1 // indirect
go.opencensus.io v0.24.0 // indirect
go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
go.uber.org/multierr v1.8.0 // indirect
go.uber.org/multierr v1.9.0 // indirect
golang.org/x/mod v0.7.0 // indirect
golang.org/x/term v0.4.0 // indirect
golang.org/x/text v0.6.0 // indirect
Expand Down
Loading