// SPDX-License-Identifier: Apache-2.0
// Copyright 2021 Authors of KubeArmor
// Package core is responsible for initiating and maintaining interactions between external entities like K8s,CRIs and internal KubeArmor entities like eBPF Monitor and Log Feeders
package core
import (
"context"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"github.com/containerd/typeurl/v2"
"google.golang.org/protobuf/proto"
"golang.org/x/exp/slices"
"github.com/kubearmor/KubeArmor/KubeArmor/common"
kl "github.com/kubearmor/KubeArmor/KubeArmor/common"
cfg "github.com/kubearmor/KubeArmor/KubeArmor/config"
kg "github.com/kubearmor/KubeArmor/KubeArmor/log"
"github.com/kubearmor/KubeArmor/KubeArmor/state"
tp "github.com/kubearmor/KubeArmor/KubeArmor/types"
"github.com/containerd/containerd/v2/core/events"
specs "github.com/opencontainers/runtime-spec/specs-go"
apievents "github.com/containerd/containerd/api/events"
task "github.com/containerd/containerd/api/services/tasks/v1"
v2 "github.com/containerd/containerd/v2/client"
"github.com/containerd/containerd/v2/pkg/namespaces"
)
// ======================== //
// == Containerd Handler == //
// ======================== //
// DefaultCaps contains all the default capabilities given to a
// container by containerd runtime
// Taken from - https://github.com/containerd/containerd/blob/main/oci/spec.go
var defaultCaps = []string{
"CAP_CHOWN",
"CAP_DAC_OVERRIDE",
"CAP_FSETID",
"CAP_FOWNER",
"CAP_MKNOD",
"CAP_NET_RAW",
"CAP_SETGID",
"CAP_SETUID",
"CAP_SETFCAP",
"CAP_SETPCAP",
"CAP_NET_BIND_SERVICE",
"CAP_SYS_CHROOT",
"CAP_KILL",
"CAP_AUDIT_WRITE",
}
// Containerd Handler
var Containerd *ContainerdHandler
// init Function
func init() {
// Spec -> google.protobuf.Any
// https://github.com/opencontainers/runtime-spec/blob/master/specs-go/config.go
const prefix = "types.containerd.io"
major := strconv.Itoa(specs.VersionMajor)
typeurl.Register(&specs.Spec{}, prefix, "opencontainers/runtime-spec", major, "Spec")
typeurl.Register(&specs.Process{}, prefix, "opencontainers/runtime-spec", major, "Process")
}
// ContainerdHandler Structure
type ContainerdHandler struct {
// container client
client *v2.Client
// context
containerd context.Context
docker context.Context
k8sEventsCh <-chan *events.Envelope
dockerEventsCh <-chan *events.Envelope
}
// NewContainerdHandler Function
func NewContainerdHandler() *ContainerdHandler {
ch := &ContainerdHandler{}
// Establish connection to containerd
client, err := v2.New(strings.TrimPrefix(cfg.GlobalCfg.CRISocket, "unix://"))
if err != nil {
kg.Errf("Unable to connect to containerd v2: %v", err)
return nil
}
ch.client = client
// Subscribe to containerd events
// docker namespace
ch.docker = context.Background()
ch.docker = namespaces.WithNamespace(context.Background(), "moby")
dockerEventsCh, _ := client.EventService().Subscribe(ch.docker, "")
ch.dockerEventsCh = dockerEventsCh
// containerd namespace
ch.containerd = namespaces.WithNamespace(context.Background(), "k8s.io")
k8sEventsCh, _ := client.EventService().Subscribe(ch.containerd, "")
ch.k8sEventsCh = k8sEventsCh
return ch
}
// Close Function
func (ch *ContainerdHandler) Close() {
if err := ch.client.Close(); err != nil {
kg.Err(err.Error())
}
}
// ==================== //
// == Container Info == //
// ==================== //
// GetContainerInfo Function
func (ch *ContainerdHandler) GetContainerInfo(ctx context.Context, containerID, nodeID string, eventpid uint32, OwnerInfo map[string]tp.PodOwner) (tp.Container, error) {
res, err := ch.client.ContainerService().Get(ctx, containerID)
if err != nil {
return tp.Container{}, err
}
// skip if pause container
if res.Labels != nil {
if containerKind, ok := res.Labels["io.cri-containerd.kind"]; ok && containerKind == "sandbox" {
return tp.Container{}, fmt.Errorf("pause container")
}
}
container := tp.Container{}
// == container base == //
container.ContainerID = res.ID
container.ContainerName = res.ID
container.NamespaceName = "Unknown"
container.EndPointName = "Unknown"
containerLabels := res.Labels
if _, ok := containerLabels["io.kubernetes.pod.namespace"]; ok { // kubernetes
if val, ok := containerLabels["io.kubernetes.pod.namespace"]; ok {
container.NamespaceName = val
}
if val, ok := containerLabels["io.kubernetes.pod.name"]; ok {
container.EndPointName = val
}
} else if val, ok := containerLabels["kubearmor.io/namespace"]; ok {
container.NamespaceName = val
} else {
container.NamespaceName = "container_namespace"
}
if len(OwnerInfo) > 0 {
if podOwnerInfo, ok := OwnerInfo[container.EndPointName]; ok {
container.Owner = podOwnerInfo
}
}
iface, err := typeurl.UnmarshalAny(res.Spec)
if err != nil {
return tp.Container{}, err
}
spec := iface.(*specs.Spec)
container.AppArmorProfile = spec.Process.ApparmorProfile
// if a container has additional caps than default, we mark it as privileged
if spec.Process.Capabilities != nil && slices.Compare(spec.Process.Capabilities.Permitted, defaultCaps) >= 0 {
container.Privileged = true
}
// == //
if eventpid == 0 {
taskReq := task.ListPidsRequest{ContainerID: container.ContainerID}
if taskRes, err := ch.client.TaskService().ListPids(ctx, &taskReq); err == nil {
if len(taskRes.Processes) == 0 {
return container, err
}
container.Pid = taskRes.Processes[0].Pid
} else {
return container, err
}
} else {
container.Pid = eventpid
}
pid := strconv.Itoa(int(container.Pid))
if data, err := os.Readlink(filepath.Join(cfg.GlobalCfg.ProcFsMount, pid, "/ns/pid")); err == nil {
if _, err := fmt.Sscanf(data, "pid:[%d]\n", &container.PidNS); err != nil {
kg.Warnf("Unable to get PidNS (%s, %s, %s)", containerID, pid, err.Error())
}
}
if data, err := os.Readlink(filepath.Join(cfg.GlobalCfg.ProcFsMount, pid, "/ns/mnt")); err == nil {
if _, err := fmt.Sscanf(data, "mnt:[%d]\n", &container.MntNS); err != nil {
kg.Warnf("Unable to get MntNS (%s, %s, %s)", containerID, pid, err.Error())
}
}
taskReq := task.ListPidsRequest{ContainerID: container.ContainerID}
if taskRes, err := ch.client.TaskService().ListPids(ctx, &taskReq); err == nil {
if len(taskRes.Processes) == 0 {
return container, err
}
pid := strconv.Itoa(int(taskRes.Processes[0].Pid))
container.Pid = taskRes.Processes[0].Pid
if data, err := os.Readlink(filepath.Join(cfg.GlobalCfg.ProcFsMount, pid, "/ns/pid")); err == nil {
if _, err := fmt.Sscanf(data, "pid:[%d]\n", &container.PidNS); err != nil {
kg.Warnf("Unable to get PidNS (%s, %s, %s)", containerID, pid, err.Error())
}
}
if data, err := os.Readlink(filepath.Join(cfg.GlobalCfg.ProcFsMount, pid, "/ns/mnt")); err == nil {
if _, err := fmt.Sscanf(data, "mnt:[%d]\n", &container.MntNS); err != nil {
kg.Warnf("Unable to get MntNS (%s, %s, %s)", containerID, pid, err.Error())
}
}
} else {
return container, err
}
// == //
if !cfg.GlobalCfg.K8sEnv {
container.ContainerImage = res.Image //+ kl.GetSHA256ofImage(inspect.Image)
container.NodeName = cfg.GlobalCfg.Host
container.NodeID = nodeID
labels := []string{}
for k, v := range res.Labels {
labels = append(labels, k+"="+v)
}
for k, v := range spec.Annotations {
labels = append(labels, k+"="+v)
}
// for policy matching
labels = append(labels, "namespaceName="+container.NamespaceName)
if _, ok := containerLabels["kubearmor.io/container.name"]; !ok {
labels = append(labels, "kubearmor.io/container.name="+container.ContainerName)
}
container.Labels = strings.Join(labels, ",")
}
// == //
return container, nil
}
// ======================= //
// == Containerd Events == //
// ======================= //
// GetContainerdContainers Function
func (ch *ContainerdHandler) GetContainerdContainers() map[string]context.Context {
containers := map[string]context.Context{}
if containerList, err := ch.client.ContainerService().List(ch.docker); err == nil {
for _, container := range containerList {
containers[container.ID] = ch.docker
}
} else {
kg.Err(err.Error())
}
if containerList, err := ch.client.ContainerService().List(ch.containerd); err == nil {
for _, container := range containerList {
containers[container.ID] = ch.containerd
}
} else {
kg.Err(err.Error())
}
return containers
}
// UpdateContainerdContainer Function
func (dm *KubeArmorDaemon) UpdateContainerdContainer(ctx context.Context, containerID string, containerPid uint32, action string) bool {
// check if Containerd exists
if Containerd == nil {
return false
}
if action == "start" {
// get container information from containerd client
container, err := Containerd.GetContainerInfo(ctx, containerID, dm.Node.NodeID, containerPid, dm.OwnerInfo)
if err != nil {
if strings.Contains(string(err.Error()), "pause container") || strings.Contains(string(err.Error()), "moby") {
kg.Debug(err.Error())
return false
}
kg.Err(err.Error())
return false
}
if container.ContainerID == "" {
return false
}
endPoint := tp.EndPoint{}
dm.ContainersLock.Lock()
if _, ok := dm.Containers[container.ContainerID]; !ok {
dm.Containers[container.ContainerID] = container
dm.ContainersLock.Unlock()
// create/update endPoint in non-k8s mode
if !dm.K8sEnabled {
endPointEvent := "ADDED"
endPointIdx := -1
containerLabels, containerIdentities := common.GetLabelsFromString(container.Labels)
dm.EndPointsLock.Lock()
// if a named endPoint exists we update
for idx, ep := range dm.EndPoints {
if container.ContainerName == ep.EndPointName || kl.MatchIdentities(ep.Identities, containerIdentities) {
endPointEvent = "UPDATED"
endPointIdx = idx
endPoint = ep
break
}
}
switch endPointEvent {
case "ADDED":
endPoint.EndPointName = container.ContainerName
endPoint.ContainerName = container.ContainerName
endPoint.NamespaceName = container.NamespaceName
endPoint.Containers = []string{container.ContainerID}
endPoint.Labels = containerLabels
endPoint.Identities = containerIdentities
endPoint.PolicyEnabled = tp.KubeArmorPolicyEnabled
endPoint.ProcessVisibilityEnabled = true
endPoint.FileVisibilityEnabled = true
endPoint.NetworkVisibilityEnabled = true
endPoint.CapabilitiesVisibilityEnabled = true
endPoint.AppArmorProfiles = []string{"kubearmor_" + container.ContainerName}
globalDefaultPosture := tp.DefaultPosture{
FileAction: cfg.GlobalCfg.DefaultFilePosture,
NetworkAction: cfg.GlobalCfg.DefaultNetworkPosture,
CapabilitiesAction: cfg.GlobalCfg.DefaultCapabilitiesPosture,
}
endPoint.DefaultPosture = globalDefaultPosture
dm.SecurityPoliciesLock.RLock()
for _, secPol := range dm.SecurityPolicies {
// required only in ADDED event, this alone will update the namespaceList for csp
updateNamespaceListforCSP(&secPol)
// match ksp || csp
if (kl.MatchIdentities(secPol.Spec.Selector.Identities, endPoint.Identities) && kl.MatchExpIdentities(secPol.Spec.Selector, endPoint.Identities)) ||
(kl.ContainsElement(secPol.Spec.Selector.NamespaceList, endPoint.NamespaceName) && kl.MatchExpIdentities(secPol.Spec.Selector, endPoint.Identities)) {
endPoint.SecurityPolicies = append(endPoint.SecurityPolicies, secPol)
}
}
dm.SecurityPoliciesLock.RUnlock()
dm.EndPoints = append(dm.EndPoints, endPoint)
case "UPDATED":
// in case of AppArmor enforcement when endPoint has to be created first
endPoint.Containers = append(endPoint.Containers, container.ContainerID)
// if this container has any additional identities, add them
endPoint.Identities = append(endPoint.Identities, containerIdentities...)
endPoint.Identities = slices.Compact(endPoint.Identities)
// add other policies
endPoint.SecurityPolicies = []tp.SecurityPolicy{}
dm.SecurityPoliciesLock.RLock()
for _, secPol := range dm.SecurityPolicies {
// match ksp || csp
if (kl.MatchIdentities(secPol.Spec.Selector.Identities, endPoint.Identities) && kl.MatchExpIdentities(secPol.Spec.Selector, endPoint.Identities)) ||
(kl.ContainsElement(secPol.Spec.Selector.NamespaceList, endPoint.NamespaceName) && kl.MatchExpIdentities(secPol.Spec.Selector, endPoint.Identities)) {
endPoint.SecurityPolicies = append(endPoint.SecurityPolicies, secPol)
}
}
dm.SecurityPoliciesLock.RUnlock()
dm.EndPoints[endPointIdx] = endPoint
}
dm.EndPointsLock.Unlock()
}
} else if dm.Containers[container.ContainerID].PidNS == 0 && dm.Containers[container.ContainerID].MntNS == 0 {
// this entry was updated by kubernetes before docker detects it
// thus, we here use the info given by kubernetes instead of the info given by docker
container.NamespaceName = dm.Containers[container.ContainerID].NamespaceName
container.EndPointName = dm.Containers[container.ContainerID].EndPointName
container.Labels = dm.Containers[container.ContainerID].Labels
container.ContainerName = dm.Containers[container.ContainerID].ContainerName
container.ContainerImage = dm.Containers[container.ContainerID].ContainerImage
container.PolicyEnabled = dm.Containers[container.ContainerID].PolicyEnabled
container.ProcessVisibilityEnabled = dm.Containers[container.ContainerID].ProcessVisibilityEnabled
container.FileVisibilityEnabled = dm.Containers[container.ContainerID].FileVisibilityEnabled
container.NetworkVisibilityEnabled = dm.Containers[container.ContainerID].NetworkVisibilityEnabled
container.CapabilitiesVisibilityEnabled = dm.Containers[container.ContainerID].CapabilitiesVisibilityEnabled
dm.Containers[container.ContainerID] = container
dm.ContainersLock.Unlock()
dm.EndPointsLock.Lock()
for idx, endPoint := range dm.EndPoints {
if endPoint.NamespaceName == container.NamespaceName && endPoint.EndPointName == container.EndPointName && kl.ContainsElement(endPoint.Containers, container.ContainerID) {
// update containers
if !kl.ContainsElement(endPoint.Containers, container.ContainerID) { // does not make sense but need to verify
dm.EndPoints[idx].Containers = append(dm.EndPoints[idx].Containers, container.ContainerID)
}
// update apparmor profiles
if !kl.ContainsElement(endPoint.AppArmorProfiles, container.AppArmorProfile) {
dm.EndPoints[idx].AppArmorProfiles = append(dm.EndPoints[idx].AppArmorProfiles, container.AppArmorProfile)
}
if container.Privileged && dm.EndPoints[idx].PrivilegedContainers != nil {
dm.EndPoints[idx].PrivilegedContainers[container.ContainerName] = struct{}{}
}
// add identities and labels if non-k8s
if !dm.K8sEnabled {
labelsSlice := strings.Split(container.Labels, ",")
for _, label := range labelsSlice {
key, value, ok := strings.Cut(label, "=")
if !ok {
continue
}
endPoint.Labels[key] = value
endPoint.Identities = append(endPoint.Identities, key+"="+value)
}
}
endPoint = dm.EndPoints[idx]
break
}
}
dm.EndPointsLock.Unlock()
} else {
dm.ContainersLock.Unlock()
return false
}
if dm.SystemMonitor != nil && cfg.GlobalCfg.Policy {
// for throttling
dm.SystemMonitor.Logger.ContainerNsKey[containerID] = common.OuterKey{
MntNs: container.MntNS,
PidNs: container.PidNS,
}
// update NsMap
dm.SystemMonitor.AddContainerIDToNsMap(containerID, container.NamespaceName, container.PidNS, container.MntNS)
dm.RuntimeEnforcer.RegisterContainer(containerID, container.PidNS, container.MntNS)
if dm.Presets != nil {
dm.Presets.RegisterContainer(container.ContainerID, container.PidNS, container.MntNS)
}
if len(endPoint.SecurityPolicies) > 0 { // struct can be empty or no policies registered for the endPoint yet
dm.Logger.UpdateSecurityPolicies("ADDED", endPoint)
if dm.RuntimeEnforcer != nil && endPoint.PolicyEnabled == tp.KubeArmorPolicyEnabled {
// enforce security policies
dm.RuntimeEnforcer.UpdateSecurityPolicies(endPoint)
}
if dm.Presets != nil && endPoint.PolicyEnabled == tp.KubeArmorPolicyEnabled {
// enforce preset rules
dm.Presets.UpdateSecurityPolicies(endPoint)
}
}
}
if cfg.GlobalCfg.StateAgent {
container.Status = "running"
go dm.StateAgent.PushContainerEvent(container, state.EventAdded)
}
dm.Logger.Printf("Detected a container (added/%.12s/pidns=%d/mntns=%d)", containerID, container.PidNS, container.MntNS)
} else if action == "destroy" {
dm.ContainersLock.Lock()
container, ok := dm.Containers[containerID]
if !ok {
dm.ContainersLock.Unlock()
return false
}
if !dm.K8sEnabled {
dm.EndPointsLock.Lock()
dm.MatchandRemoveContainerFromEndpoint(containerID)
dm.EndPointsLock.Unlock()
}
delete(dm.Containers, containerID)
dm.ContainersLock.Unlock()
// delete endpoint if no security rules and containers
if !dm.K8sEnabled {
idx := 0
endpointsLength := len(dm.EndPoints)
for idx < endpointsLength {
endpoint := dm.EndPoints[idx]
if container.NamespaceName == endpoint.NamespaceName && container.ContainerName == endpoint.EndPointName &&
len(endpoint.SecurityPolicies) == 0 && len(endpoint.Containers) == 0 {
dm.EndPoints = append(dm.EndPoints[:idx], dm.EndPoints[idx+1:]...)
endpointsLength--
idx--
}
idx++
}
}
dm.EndPointsLock.Lock()
for idx, endPoint := range dm.EndPoints {
if endPoint.NamespaceName == container.NamespaceName && endPoint.EndPointName == container.EndPointName && kl.ContainsElement(endPoint.Containers, container.ContainerID) {
// update apparmor profiles
for idxA, profile := range endPoint.AppArmorProfiles {
if profile == container.AppArmorProfile {
dm.EndPoints[idx].AppArmorProfiles = append(dm.EndPoints[idx].AppArmorProfiles[:idxA], dm.EndPoints[idx].AppArmorProfiles[idxA+1:]...)
break
}
}
break
}
}
dm.EndPointsLock.Unlock()
if dm.SystemMonitor != nil && cfg.GlobalCfg.Policy {
outkey := dm.SystemMonitor.Logger.ContainerNsKey[containerID]
dm.Logger.DeleteAlertMapKey(outkey)
delete(dm.SystemMonitor.Logger.ContainerNsKey, containerID)
// update NsMap
dm.SystemMonitor.DeleteContainerIDFromNsMap(containerID, container.NamespaceName, container.PidNS, container.MntNS)
dm.RuntimeEnforcer.UnregisterContainer(containerID)
if dm.Presets != nil {
dm.Presets.UnregisterContainer(containerID)
}
}
if cfg.GlobalCfg.StateAgent {
container.Status = "terminated"
go dm.StateAgent.PushContainerEvent(container, state.EventDeleted)
}
dm.Logger.Printf("Detected a container (removed/%.12s/pidns=%d/mntns=%d)", containerID, container.PidNS, container.MntNS)
}
return true
}
// MonitorContainerdEvents Function
func (dm *KubeArmorDaemon) MonitorContainerdEvents() {
dm.WgDaemon.Add(1)
defer dm.WgDaemon.Done()
Containerd = NewContainerdHandler()
// check if Containerd exists
if Containerd == nil {
return
}
dm.Logger.Print("Started to monitor Containerd events")
containers := Containerd.GetContainerdContainers()
if len(containers) > 0 {
for containerID, context := range containers {
if !dm.UpdateContainerdContainer(context, containerID, 0, "start") {
continue
}
}
}
for {
select {
case <-StopChan:
return
case envelope := <-Containerd.k8sEventsCh:
dm.handleContainerdEvent(envelope, Containerd.containerd)
case envelope := <-Containerd.dockerEventsCh:
dm.handleContainerdEvent(envelope, Containerd.docker)
}
}
}
func (dm *KubeArmorDaemon) handleContainerdEvent(envelope *events.Envelope, context context.Context) {
if envelope == nil {
return
}
// Handle the different event types
switch envelope.Topic {
case "/containers/delete":
deleteContainer := &apievents.ContainerDelete{}
err := proto.Unmarshal(envelope.Event.GetValue(), deleteContainer)
if err != nil {
kg.Errf("failed to unmarshal container's delete event: %v", err)
}
dm.UpdateContainerdContainer(context, deleteContainer.GetID(), 0, "destroy")
case "/tasks/start":
startTask := &apievents.TaskStart{}
err := proto.Unmarshal(envelope.Event.GetValue(), startTask)
if err != nil {
kg.Errf("failed to unmarshal container's start task: %v", err)
}
dm.UpdateContainerdContainer(context, startTask.GetContainerID(), startTask.GetPid(), "start")
case "/tasks/exit":
exitTask := &apievents.TaskStart{}
err := proto.Unmarshal(envelope.Event.GetValue(), exitTask)
if err != nil {
kg.Errf("failed to unmarshal container's exit task: %v", err)
}
dm.ContainersLock.RLock()
pid := dm.Containers[exitTask.GetContainerID()].Pid
dm.ContainersLock.RUnlock()
if pid == exitTask.GetPid() {
dm.UpdateContainerdContainer(context, exitTask.GetContainerID(), pid, "destroy")
}
}
}
// SPDX-License-Identifier: Apache-2.0
// Copyright 2021 Authors of KubeArmor
package core
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strconv"
"time"
"github.com/kubearmor/KubeArmor/KubeArmor/common"
kl "github.com/kubearmor/KubeArmor/KubeArmor/common"
cfg "github.com/kubearmor/KubeArmor/KubeArmor/config"
kg "github.com/kubearmor/KubeArmor/KubeArmor/log"
tp "github.com/kubearmor/KubeArmor/KubeArmor/types"
spec "github.com/opencontainers/runtime-spec/specs-go"
"google.golang.org/grpc"
pb "k8s.io/cri-api/pkg/apis/runtime/v1"
)
// CrioHandler Structure
type CrioHandler struct {
// connection
conn *grpc.ClientConn
// crio client
client pb.RuntimeServiceClient
// containers is a map with empty value to have lookups in constant time
containers map[string]struct{}
}
// CrioContainerInfo struct corresponds to CRI-O's container info returned
// with container status
type CrioContainerInfo struct {
SandboxID string `json:"sandboxID"`
Pid int `json:"pid"`
RuntimeSpec spec.Spec `json:"runtimeSpec"`
Privileged bool `json:"privileged"`
}
// Crio Handler
var Crio *CrioHandler
// NewCrioHandler Function creates a new Crio handler
func NewCrioHandler() *CrioHandler {
ch := &CrioHandler{}
conn, err := grpc.Dial(cfg.GlobalCfg.CRISocket, grpc.WithInsecure())
if err != nil {
return nil
}
ch.conn = conn
// The runtime service client can be used for all RPCs
ch.client = pb.NewRuntimeServiceClient(ch.conn)
ch.containers = make(map[string]struct{})
return ch
}
// Close the connection
func (ch *CrioHandler) Close() {
if ch.conn != nil {
if err := ch.conn.Close(); err != nil {
kg.Err(err.Error())
}
}
}
// ==================== //
// == Container Info == //
// ==================== //
// GetContainerInfo Function gets info of a particular container
func (ch *CrioHandler) GetContainerInfo(ctx context.Context, containerID, nodeID string, OwnerInfo map[string]tp.PodOwner) (tp.Container, error) {
// request to get status of specified container
// verbose has to be true to retrieve additional CRI specific info
req := &pb.ContainerStatusRequest{
ContainerId: containerID,
Verbose: true,
}
res, err := ch.client.ContainerStatus(ctx, req)
if err != nil {
return tp.Container{}, err
}
container := tp.Container{}
// == container base == //
resContainerStatus := res.Status
container.ContainerID = resContainerStatus.Id
container.ContainerName = resContainerStatus.Metadata.Name
container.NamespaceName = "Unknown"
container.EndPointName = "Unknown"
// check container labels
containerLabels := resContainerStatus.Labels
if val, ok := containerLabels["io.kubernetes.pod.namespace"]; ok {
container.NamespaceName = val
}
if val, ok := containerLabels["io.kubernetes.pod.name"]; ok {
container.EndPointName = val
}
if len(OwnerInfo) > 0 {
if podOwnerInfo, ok := OwnerInfo[container.EndPointName]; ok {
container.Owner = podOwnerInfo
}
}
if !cfg.GlobalCfg.K8sEnv {
container.NodeName = cfg.GlobalCfg.Host
container.NodeID = nodeID
}
// extracting the runtime specific "info"
var containerInfo CrioContainerInfo
err = json.Unmarshal([]byte(res.Info["info"]), &containerInfo)
if err != nil {
return tp.Container{}, err
}
// path to container's root storage
container.AppArmorProfile = containerInfo.RuntimeSpec.Process.ApparmorProfile
container.Privileged = containerInfo.Privileged
pid := strconv.Itoa(containerInfo.Pid)
if data, err := os.Readlink(filepath.Join(cfg.GlobalCfg.ProcFsMount, pid, "/ns/pid")); err == nil {
if _, err := fmt.Sscanf(data, "pid:[%d]\n", &container.PidNS); err != nil {
kg.Warnf("Unable to get PidNS (%s, %s, %s)", containerID, pid, err.Error())
}
} else {
return container, err
}
if data, err := os.Readlink(filepath.Join(cfg.GlobalCfg.ProcFsMount, pid, "/ns/mnt")); err == nil {
if _, err := fmt.Sscanf(data, "mnt:[%d]\n", &container.MntNS); err != nil {
kg.Warnf("Unable to get MntNS (%s, %s, %s)", containerID, pid, err.Error())
}
} else {
return container, err
}
return container, nil
}
// ================= //
// == CRIO Events == //
// ================= //
// GetCrioContainers Function gets IDs of all containers
func (ch *CrioHandler) GetCrioContainers() (map[string]struct{}, error) {
containers := make(map[string]struct{})
var err error
req := pb.ListContainersRequest{}
if containerList, err := ch.client.ListContainers(context.Background(), &req, grpc.MaxCallRecvMsgSize(kl.DefaultMaxRecvMaxSize)); err == nil {
for _, container := range containerList.Containers {
containers[container.Id] = struct{}{}
}
return containers, nil
}
return nil, err
}
// GetNewCrioContainers Function gets new crio containers
func (ch *CrioHandler) GetNewCrioContainers(containers map[string]struct{}) map[string]struct{} {
newContainers := make(map[string]struct{})
for activeContainerID := range containers {
if _, ok := ch.containers[activeContainerID]; !ok {
newContainers[activeContainerID] = struct{}{}
}
}
return newContainers
}
// GetDeletedCrioContainers Function gets deleted crio containers
func (ch *CrioHandler) GetDeletedCrioContainers(containers map[string]struct{}) map[string]struct{} {
deletedContainers := make(map[string]struct{})
for globalContainerID := range ch.containers {
if _, ok := containers[globalContainerID]; !ok {
deletedContainers[globalContainerID] = struct{}{}
delete(ch.containers, globalContainerID)
}
}
ch.containers = containers
return deletedContainers
}
// UpdateCrioContainer Function
func (dm *KubeArmorDaemon) UpdateCrioContainer(ctx context.Context, containerID, action string) bool {
if Crio == nil {
return false
}
if action == "start" {
// get container info from client
container, err := Crio.GetContainerInfo(ctx, containerID, dm.Node.NodeID, dm.OwnerInfo)
if err != nil {
return false
}
if container.ContainerID == "" {
return false
}
endpoint := tp.EndPoint{}
dm.ContainersLock.Lock()
if _, ok := dm.Containers[container.ContainerID]; !ok {
dm.Containers[container.ContainerID] = container
dm.ContainersLock.Unlock()
} else if dm.Containers[container.ContainerID].PidNS == 0 && dm.Containers[container.ContainerID].MntNS == 0 {
container.NamespaceName = dm.Containers[container.ContainerID].NamespaceName
container.EndPointName = dm.Containers[container.ContainerID].EndPointName
container.Labels = dm.Containers[container.ContainerID].Labels
container.ContainerName = dm.Containers[container.ContainerID].ContainerName
container.ContainerImage = dm.Containers[container.ContainerID].ContainerImage
container.PolicyEnabled = dm.Containers[container.ContainerID].PolicyEnabled
container.ProcessVisibilityEnabled = dm.Containers[container.ContainerID].ProcessVisibilityEnabled
container.FileVisibilityEnabled = dm.Containers[container.ContainerID].FileVisibilityEnabled
container.NetworkVisibilityEnabled = dm.Containers[container.ContainerID].NetworkVisibilityEnabled
container.CapabilitiesVisibilityEnabled = dm.Containers[container.ContainerID].CapabilitiesVisibilityEnabled
dm.Containers[container.ContainerID] = container
dm.ContainersLock.Unlock()
dm.EndPointsLock.Lock()
for idx, endPoint := range dm.EndPoints {
if endPoint.NamespaceName == container.NamespaceName && endPoint.EndPointName == container.EndPointName && kl.ContainsElement(endPoint.Containers, container.ContainerID) {
// update apparmor profiles
if !kl.ContainsElement(endPoint.AppArmorProfiles, container.AppArmorProfile) {
dm.EndPoints[idx].AppArmorProfiles = append(dm.EndPoints[idx].AppArmorProfiles, container.AppArmorProfile)
}
if container.Privileged && dm.EndPoints[idx].PrivilegedContainers != nil {
dm.EndPoints[idx].PrivilegedContainers[container.ContainerName] = struct{}{}
}
endpoint = dm.EndPoints[idx]
break
}
}
dm.EndPointsLock.Unlock()
} else {
dm.ContainersLock.Unlock()
return false
}
if dm.SystemMonitor != nil && cfg.GlobalCfg.Policy {
// for throttling
dm.SystemMonitor.Logger.ContainerNsKey[containerID] = common.OuterKey{
MntNs: container.MntNS,
PidNs: container.PidNS,
}
// update NsMap
dm.SystemMonitor.AddContainerIDToNsMap(containerID, container.NamespaceName, container.PidNS, container.MntNS)
dm.RuntimeEnforcer.RegisterContainer(containerID, container.PidNS, container.MntNS)
if dm.Presets != nil {
dm.Presets.RegisterContainer(containerID, container.PidNS, container.MntNS)
}
if len(endpoint.SecurityPolicies) > 0 { // struct can be empty or no policies registered for the endpoint yet
dm.Logger.UpdateSecurityPolicies("ADDED", endpoint)
if dm.RuntimeEnforcer != nil && endpoint.PolicyEnabled == tp.KubeArmorPolicyEnabled {
// enforce security policies
dm.RuntimeEnforcer.UpdateSecurityPolicies(endpoint)
}
if dm.Presets != nil && endpoint.PolicyEnabled == tp.KubeArmorPolicyEnabled {
// enforce preset rules
dm.Presets.UpdateSecurityPolicies(endpoint)
}
}
}
if !dm.K8sEnabled {
dm.ContainersLock.Lock()
dm.EndPointsLock.Lock()
dm.MatchandUpdateContainerSecurityPolicies(containerID)
dm.EndPointsLock.Unlock()
dm.ContainersLock.Unlock()
}
dm.Logger.Printf("Detected a container (added/%.12s)", containerID)
} else if action == "destroy" {
dm.ContainersLock.Lock()
container, ok := dm.Containers[containerID]
if !ok {
dm.ContainersLock.Unlock()
return false
}
if !dm.K8sEnabled {
dm.EndPointsLock.Lock()
dm.MatchandRemoveContainerFromEndpoint(containerID)
dm.EndPointsLock.Unlock()
}
delete(dm.Containers, containerID)
dm.ContainersLock.Unlock()
dm.EndPointsLock.Lock()
for idx, endPoint := range dm.EndPoints {
if endPoint.NamespaceName == container.NamespaceName && endPoint.EndPointName == container.EndPointName && kl.ContainsElement(endPoint.Containers, container.ContainerID) {
// update apparmor profiles
for idxA, profile := range endPoint.AppArmorProfiles {
if profile == container.AppArmorProfile {
dm.EndPoints[idx].AppArmorProfiles = append(dm.EndPoints[idx].AppArmorProfiles[:idxA], dm.EndPoints[idx].AppArmorProfiles[idxA+1:]...)
break
}
}
break
}
}
dm.EndPointsLock.Unlock()
if dm.SystemMonitor != nil && cfg.GlobalCfg.Policy {
outkey := dm.SystemMonitor.Logger.ContainerNsKey[containerID]
dm.Logger.DeleteAlertMapKey(outkey)
delete(dm.SystemMonitor.Logger.ContainerNsKey, containerID)
// update NsMap
dm.SystemMonitor.DeleteContainerIDFromNsMap(containerID, container.NamespaceName, container.PidNS, container.MntNS)
dm.RuntimeEnforcer.UnregisterContainer(containerID)
}
dm.Logger.Printf("Detected a container (removed/%.12s)", containerID)
}
return true
}
// MonitorCrioEvents Function
func (dm *KubeArmorDaemon) MonitorCrioEvents() {
dm.WgDaemon.Add(1)
defer dm.WgDaemon.Done()
Crio = NewCrioHandler()
// check if Crio exists
if Crio == nil {
return
}
dm.Logger.Print("Started to monitor CRI-O events")
for {
select {
case <-StopChan:
return
default:
containers, err := Crio.GetCrioContainers()
if err != nil {
return
}
invalidContainers := []string{}
newContainers := Crio.GetNewCrioContainers(containers)
deletedContainers := Crio.GetDeletedCrioContainers(containers)
if len(newContainers) > 0 {
for containerID := range newContainers {
if !dm.UpdateCrioContainer(context.Background(), containerID, "start") {
invalidContainers = append(invalidContainers, containerID)
}
}
}
for _, invalidContainerID := range invalidContainers {
delete(Crio.containers, invalidContainerID)
}
if len(deletedContainers) > 0 {
for containerID := range deletedContainers {
dm.UpdateCrioContainer(context.Background(), containerID, "destroy")
}
}
}
time.Sleep(time.Millisecond * 50)
}
}
// SPDX-License-Identifier: Apache-2.0
// Copyright 2021 Authors of KubeArmor
package core
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"slices"
"strconv"
"strings"
"time"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/events"
"github.com/docker/docker/client"
"github.com/kubearmor/KubeArmor/KubeArmor/common"
kl "github.com/kubearmor/KubeArmor/KubeArmor/common"
cfg "github.com/kubearmor/KubeArmor/KubeArmor/config"
kg "github.com/kubearmor/KubeArmor/KubeArmor/log"
"github.com/kubearmor/KubeArmor/KubeArmor/state"
tp "github.com/kubearmor/KubeArmor/KubeArmor/types"
)
// ==================== //
// == Docker Handler == //
// ==================== //
// Docker Handler
var Docker *DockerHandler
// DockerVersion Structure
type DockerVersion struct {
APIVersion string `json:"ApiVersion"`
}
// DockerHandler Structure
type DockerHandler struct {
DockerClient *client.Client
Version DockerVersion
// needed for container info
NodeIP string
}
// NewDockerHandler Function
func NewDockerHandler() (*DockerHandler, error) {
docker := &DockerHandler{}
// try to create a new docker client
// If env DOCKER_API_VERSION set - NegotiateAPIVersion() won't do anything
DockerClient, err := client.NewClientWithOpts(client.FromEnv)
if err != nil {
return nil, err
}
DockerClient.NegotiateAPIVersion(context.Background())
clientVersion := DockerClient.ClientVersion()
kg.Printf("Verifying Docker API client version: %s", clientVersion)
serverVersion, err := DockerClient.ServerVersion(context.Background())
if err != nil {
return nil, err
}
if clientVersion != serverVersion.APIVersion {
kg.Warnf("Docker client (%s) and Docker server (%s) API versions don't match", clientVersion, serverVersion.APIVersion)
}
docker.DockerClient = DockerClient
docker.NodeIP = kl.GetExternalIPAddr()
kg.Printf("Initialized Docker Handler (version: %s)", clientVersion)
return docker, nil
}
// Close Function
func (dh *DockerHandler) Close() {
if dh.DockerClient != nil {
if err := dh.DockerClient.Close(); err != nil {
kg.Err(err.Error())
}
}
}
// ==================== //
// == Container Info == //
// ==================== //
// GetContainerInfo Function
func (dh *DockerHandler) GetContainerInfo(containerID, nodeID string, OwnerInfo map[string]tp.PodOwner) (tp.Container, error) {
if dh.DockerClient == nil {
return tp.Container{}, errors.New("no docker client")
}
inspect, err := dh.DockerClient.ContainerInspect(context.Background(), containerID)
if err != nil {
return tp.Container{}, err
}
container := tp.Container{}
// == container base == //
container.ContainerID = inspect.ID
container.ContainerName = strings.TrimLeft(inspect.Name, "/")
container.NamespaceName = "Unknown"
container.EndPointName = "Unknown"
containerLabels := make(map[string]string)
containerLabels = inspect.Config.Labels
if _, ok := containerLabels["io.kubernetes.pod.namespace"]; ok { // kubernetes
if val, ok := containerLabels["io.kubernetes.pod.namespace"]; ok {
container.NamespaceName = val
}
if val, ok := containerLabels["io.kubernetes.pod.name"]; ok {
container.EndPointName = val
}
} else if val, ok := containerLabels["kubearmor.io/namespace"]; ok {
container.NamespaceName = val
} else {
container.NamespaceName = "container_namespace"
}
if len(OwnerInfo) > 0 {
if podOwnerInfo, ok := OwnerInfo[container.EndPointName]; ok {
container.Owner = podOwnerInfo
}
}
container.AppArmorProfile = inspect.AppArmorProfile
if inspect.HostConfig.Privileged ||
len(inspect.HostConfig.CapAdd) > 0 {
container.Privileged = inspect.HostConfig.Privileged
}
// == //
pid := strconv.Itoa(inspect.State.Pid)
if data, err := os.Readlink(filepath.Join(cfg.GlobalCfg.ProcFsMount, pid, "/ns/pid")); err == nil {
if _, err := fmt.Sscanf(data, "pid:[%d]\n", &container.PidNS); err != nil {
kg.Warnf("Unable to get PidNS (%s, %s, %s)", containerID, pid, err.Error())
}
}
if data, err := os.Readlink(filepath.Join(cfg.GlobalCfg.ProcFsMount, pid, "/ns/mnt")); err == nil {
if _, err := fmt.Sscanf(data, "mnt:[%d]\n", &container.MntNS); err != nil {
kg.Warnf("Unable to get MntNS (%s, %s, %s)", containerID, pid, err.Error())
}
}
// == //
if !cfg.GlobalCfg.K8sEnv {
container.ContainerImage = inspect.Config.Image //+ kl.GetSHA256ofImage(inspect.Image)
container.NodeName = cfg.GlobalCfg.Host
container.NodeID = nodeID
labels := []string{}
for k, v := range containerLabels {
labels = append(labels, k+"="+v)
}
// for policy matching
labels = append(labels, "namespaceName="+container.NamespaceName)
if _, ok := containerLabels["kubearmor.io/container.name"]; !ok {
labels = append(labels, "kubearmor.io/container.name="+container.ContainerName)
}
container.Labels = strings.Join(labels, ",")
var podIP string
if inspect.HostConfig != nil {
if inspect.HostConfig.NetworkMode.IsNone() || inspect.HostConfig.NetworkMode.IsContainer() {
podIP = ""
} else if inspect.HostConfig.NetworkMode.IsHost() {
podIP = dh.NodeIP
} else {
// user defined network OR swarm mode
networkName := inspect.HostConfig.NetworkMode.NetworkName()
networkInfo, ok := inspect.NetworkSettings.Networks[networkName]
if ok && networkInfo != nil {
podIP = networkInfo.IPAddress
}
}
}
container.ContainerIP = podIP
// time format used by docker engine is RFC3339Nano
lastUpdatedAt, err := time.Parse(time.RFC3339Nano, inspect.State.StartedAt)
if err == nil {
container.LastUpdatedAt = lastUpdatedAt.UTC().String()
}
// finished at is IsZero until a container exits
timeFinished, err := time.Parse(time.RFC3339Nano, inspect.State.FinishedAt)
if err == nil && !timeFinished.IsZero() && timeFinished.After(lastUpdatedAt) {
lastUpdatedAt = timeFinished
}
}
return container, nil
}
// ========================== //
// == Docker Event Channel == //
// ========================== //
// GetEventChannel Function
func (dh *DockerHandler) GetEventChannel(ctx context.Context, StopChan <-chan struct{}) <-chan events.Message {
if dh.DockerClient != nil {
eventBuffer := make(chan events.Message, 256)
go func() {
eventStream, _ := dh.DockerClient.Events(ctx, events.ListOptions{})
defer close(eventBuffer)
for event := range eventStream {
select {
case eventBuffer <- event:
case <-ctx.Done():
return
case <-StopChan:
return
default:
kg.Warnf("Docker channel full.")
}
}
}()
return eventBuffer
}
return nil
}
// =================== //
// == Docker Events == //
// =================== //
// SetContainerVisibility function enables visibility flag arguments for un-orchestrated container
func (dm *KubeArmorDaemon) SetContainerVisibility(containerID string) {
// get container information from docker client
container, err := Docker.GetContainerInfo(containerID, dm.Node.NodeID, dm.OwnerInfo)
if err != nil {
return
}
if strings.Contains(cfg.GlobalCfg.Visibility, "process") {
container.ProcessVisibilityEnabled = true
}
if strings.Contains(cfg.GlobalCfg.Visibility, "file") {
container.FileVisibilityEnabled = true
}
if strings.Contains(cfg.GlobalCfg.Visibility, "network") {
container.NetworkVisibilityEnabled = true
}
if strings.Contains(cfg.GlobalCfg.Visibility, "capabilities") {
container.CapabilitiesVisibilityEnabled = true
}
container.EndPointName = container.ContainerName
container.NamespaceName = "container_namespace"
dm.Containers[container.ContainerID] = container
}
// GetAlreadyDeployedDockerContainers Function
func (dm *KubeArmorDaemon) GetAlreadyDeployedDockerContainers() {
// check if Docker exists else instantiate
if Docker == nil {
var err error
Docker, err = NewDockerHandler()
if err != nil {
dm.Logger.Errf("Failed to create new Docker client: %s", err.Error())
return
}
}
if containerList, err := Docker.DockerClient.ContainerList(context.Background(), container.ListOptions{}); err == nil {
for _, dcontainer := range containerList {
// get container information from docker client
container, err := Docker.GetContainerInfo(dcontainer.ID, dm.Node.NodeID, dm.OwnerInfo)
if err != nil {
continue
}
if container.ContainerID == "" {
continue
}
endPoint := tp.EndPoint{}
if dcontainer.State == "running" {
dm.ContainersLock.Lock()
if _, ok := dm.Containers[container.ContainerID]; !ok {
dm.Containers[container.ContainerID] = container
dm.ContainersLock.Unlock()
// create/update endpoint in non-k8s mode
if !dm.K8sEnabled {
endPointEvent := "ADDED"
endPointIdx := -1
containerLabels, containerIdentities := common.GetLabelsFromString(container.Labels)
dm.EndPointsLock.Lock()
// if a named endpoint exists we update
for idx, ep := range dm.EndPoints {
if container.ContainerName == ep.EndPointName || kl.MatchIdentities(ep.Identities, containerIdentities) {
endPointEvent = "UPDATED"
endPointIdx = idx
endPoint = ep
break
}
}
switch endPointEvent {
case "ADDED":
endPoint.EndPointName = container.ContainerName
endPoint.ContainerName = container.ContainerName
endPoint.NamespaceName = container.NamespaceName
endPoint.Containers = []string{container.ContainerID}
endPoint.Labels = containerLabels
endPoint.Identities = containerIdentities
endPoint.PolicyEnabled = tp.KubeArmorPolicyEnabled
endPoint.ProcessVisibilityEnabled = true
endPoint.FileVisibilityEnabled = true
endPoint.NetworkVisibilityEnabled = true
endPoint.CapabilitiesVisibilityEnabled = true
endPoint.AppArmorProfiles = []string{"kubearmor_" + container.ContainerName}
globalDefaultPosture := tp.DefaultPosture{
FileAction: cfg.GlobalCfg.DefaultFilePosture,
NetworkAction: cfg.GlobalCfg.DefaultNetworkPosture,
CapabilitiesAction: cfg.GlobalCfg.DefaultCapabilitiesPosture,
}
endPoint.DefaultPosture = globalDefaultPosture
dm.SecurityPoliciesLock.RLock()
for _, secPol := range dm.SecurityPolicies {
// required only in ADDED event, this alone will update the namespaceList for csp
updateNamespaceListforCSP(&secPol)
// match ksp || csp
if (kl.MatchIdentities(secPol.Spec.Selector.Identities, endPoint.Identities) && kl.MatchExpIdentities(secPol.Spec.Selector, endPoint.Identities)) ||
(kl.ContainsElement(secPol.Spec.Selector.NamespaceList, endPoint.NamespaceName) && kl.MatchExpIdentities(secPol.Spec.Selector, endPoint.Identities)) {
endPoint.SecurityPolicies = append(endPoint.SecurityPolicies, secPol)
}
}
dm.SecurityPoliciesLock.RUnlock()
dm.EndPoints = append(dm.EndPoints, endPoint)
case "UPDATED":
// in case of AppArmor enforcement when endpoint has to be created first
endPoint.Containers = append(endPoint.Containers, container.ContainerID)
// if this container has any additional identities, add them
endPoint.Identities = append(endPoint.Identities, containerIdentities...)
endPoint.Identities = slices.Compact(endPoint.Identities)
// add other policies
endPoint.SecurityPolicies = []tp.SecurityPolicy{}
dm.SecurityPoliciesLock.RLock()
for _, secPol := range dm.SecurityPolicies {
// match ksp || csp
if (kl.MatchIdentities(secPol.Spec.Selector.Identities, endPoint.Identities) && kl.MatchExpIdentities(secPol.Spec.Selector, endPoint.Identities)) ||
(kl.ContainsElement(secPol.Spec.Selector.NamespaceList, endPoint.NamespaceName) && kl.MatchExpIdentities(secPol.Spec.Selector, endPoint.Identities)) {
endPoint.SecurityPolicies = append(endPoint.SecurityPolicies, secPol)
}
}
dm.SecurityPoliciesLock.RUnlock()
dm.EndPoints[endPointIdx] = endPoint
}
dm.EndPointsLock.Unlock()
}
} else if dm.Containers[container.ContainerID].PidNS == 0 && dm.Containers[container.ContainerID].MntNS == 0 {
// this entry was updated by kubernetes before docker detects it
// thus, we here use the info given by kubernetes instead of the info given by docker
container.NamespaceName = dm.Containers[container.ContainerID].NamespaceName
container.EndPointName = dm.Containers[container.ContainerID].EndPointName
container.Labels = dm.Containers[container.ContainerID].Labels
container.ContainerName = dm.Containers[container.ContainerID].ContainerName
container.ContainerImage = dm.Containers[container.ContainerID].ContainerImage
container.PolicyEnabled = dm.Containers[container.ContainerID].PolicyEnabled
container.ProcessVisibilityEnabled = dm.Containers[container.ContainerID].ProcessVisibilityEnabled
container.FileVisibilityEnabled = dm.Containers[container.ContainerID].FileVisibilityEnabled
container.NetworkVisibilityEnabled = dm.Containers[container.ContainerID].NetworkVisibilityEnabled
container.CapabilitiesVisibilityEnabled = dm.Containers[container.ContainerID].CapabilitiesVisibilityEnabled
dm.Containers[container.ContainerID] = container
dm.ContainersLock.Unlock()
dm.EndPointsLock.Lock()
for idx, endPoint := range dm.EndPoints {
if endPoint.NamespaceName == container.NamespaceName && endPoint.EndPointName == container.EndPointName && kl.ContainsElement(endPoint.Containers, container.ContainerID) {
// update apparmor profiles
if !kl.ContainsElement(endPoint.AppArmorProfiles, container.AppArmorProfile) {
dm.EndPoints[idx].AppArmorProfiles = append(dm.EndPoints[idx].AppArmorProfiles, container.AppArmorProfile)
}
if container.Privileged && dm.EndPoints[idx].PrivilegedContainers != nil {
dm.EndPoints[idx].PrivilegedContainers[container.ContainerName] = struct{}{}
}
endPoint = dm.EndPoints[idx]
break
}
}
dm.EndPointsLock.Unlock()
} else {
dm.ContainersLock.Unlock()
continue
}
// check for unorchestrated docker containers
if !dm.K8sEnabled {
dm.ContainersLock.Lock()
dm.SetContainerVisibility(dcontainer.ID)
container = dm.Containers[dcontainer.ID]
dm.ContainersLock.Unlock()
}
if cfg.GlobalCfg.StateAgent {
go dm.StateAgent.PushContainerEvent(container, state.EventAdded)
}
if dm.SystemMonitor != nil && cfg.GlobalCfg.Policy {
// for throttling
dm.SystemMonitor.Logger.ContainerNsKey[container.ContainerID] = common.OuterKey{
MntNs: container.MntNS,
PidNs: container.PidNS,
}
// update NsMap
dm.SystemMonitor.AddContainerIDToNsMap(container.ContainerID, container.NamespaceName, container.PidNS, container.MntNS)
dm.RuntimeEnforcer.RegisterContainer(container.ContainerID, container.PidNS, container.MntNS)
if dm.Presets != nil {
dm.Presets.RegisterContainer(container.ContainerID, container.PidNS, container.MntNS)
}
if len(endPoint.SecurityPolicies) > 0 { // struct can be empty or no policies registered for the endpoint yet
dm.Logger.UpdateSecurityPolicies("ADDED", endPoint)
if dm.RuntimeEnforcer != nil && endPoint.PolicyEnabled == tp.KubeArmorPolicyEnabled {
// enforce security policies
dm.RuntimeEnforcer.UpdateSecurityPolicies(endPoint)
}
if dm.Presets != nil && endPoint.PolicyEnabled == tp.KubeArmorPolicyEnabled {
// enforce preset rules
dm.Presets.UpdateSecurityPolicies(endPoint)
}
}
}
dm.Logger.Printf("Detected a container (added/%.12s)", container.ContainerID)
}
}
} else {
dm.Logger.Warnf("Error while listing containers: %s", err)
}
}
// UpdateDockerContainer Function
func (dm *KubeArmorDaemon) UpdateDockerContainer(containerID, action string) {
// check if Docker exists
if Docker == nil {
return
}
container := tp.Container{}
if action == "start" {
var err error
// get container information from docker client
container, err = Docker.GetContainerInfo(containerID, dm.Node.NodeID, dm.OwnerInfo)
if err != nil {
return
}
if container.ContainerID == "" {
return
}
endPoint := tp.EndPoint{}
dm.ContainersLock.Lock()
if _, ok := dm.Containers[containerID]; !ok {
dm.Containers[containerID] = container
dm.ContainersLock.Unlock()
// create/update endpoint in non-k8s mode
if !dm.K8sEnabled {
endPointEvent := "ADDED"
endPointIdx := -1
containerLabels, containerIdentities := common.GetLabelsFromString(container.Labels)
dm.EndPointsLock.Lock()
// if a named endpoint exists we update
for idx, ep := range dm.EndPoints {
if container.ContainerName == ep.EndPointName || kl.MatchIdentities(ep.Identities, containerIdentities) {
endPointEvent = "UPDATED"
endPointIdx = idx
endPoint = ep
break
}
}
switch endPointEvent {
case "ADDED":
endPoint.EndPointName = container.ContainerName
endPoint.ContainerName = container.ContainerName
endPoint.NamespaceName = container.NamespaceName
endPoint.Containers = []string{container.ContainerID}
endPoint.Labels = containerLabels
endPoint.Identities = containerIdentities
endPoint.PolicyEnabled = tp.KubeArmorPolicyEnabled
endPoint.ProcessVisibilityEnabled = true
endPoint.FileVisibilityEnabled = true
endPoint.NetworkVisibilityEnabled = true
endPoint.CapabilitiesVisibilityEnabled = true
endPoint.AppArmorProfiles = []string{"kubearmor_" + container.ContainerName}
globalDefaultPosture := tp.DefaultPosture{
FileAction: cfg.GlobalCfg.DefaultFilePosture,
NetworkAction: cfg.GlobalCfg.DefaultNetworkPosture,
CapabilitiesAction: cfg.GlobalCfg.DefaultCapabilitiesPosture,
}
endPoint.DefaultPosture = globalDefaultPosture
dm.SecurityPoliciesLock.RLock()
for _, secPol := range dm.SecurityPolicies {
updateNamespaceListforCSP(&secPol)
// match ksp || csp
if (kl.MatchIdentities(secPol.Spec.Selector.Identities, endPoint.Identities) && kl.MatchExpIdentities(secPol.Spec.Selector, endPoint.Identities)) ||
(kl.ContainsElement(secPol.Spec.Selector.NamespaceList, endPoint.NamespaceName) && kl.MatchExpIdentities(secPol.Spec.Selector, endPoint.Identities)) {
endPoint.SecurityPolicies = append(endPoint.SecurityPolicies, secPol)
}
}
dm.SecurityPoliciesLock.RUnlock()
dm.EndPoints = append(dm.EndPoints, endPoint)
case "UPDATED":
// in case of AppArmor enforcement when endpoint has to be created first
endPoint.Containers = append(endPoint.Containers, container.ContainerID)
// if this container has any additional identities, add them
endPoint.Identities = append(endPoint.Identities, containerIdentities...)
endPoint.Identities = slices.Compact(endPoint.Identities)
// add other policies
endPoint.SecurityPolicies = []tp.SecurityPolicy{}
dm.SecurityPoliciesLock.RLock()
for _, secPol := range dm.SecurityPolicies {
// match ksp || csp
if (kl.MatchIdentities(secPol.Spec.Selector.Identities, endPoint.Identities) && kl.MatchExpIdentities(secPol.Spec.Selector, endPoint.Identities)) ||
(kl.ContainsElement(secPol.Spec.Selector.NamespaceList, endPoint.NamespaceName) && kl.MatchExpIdentities(secPol.Spec.Selector, endPoint.Identities)) {
endPoint.SecurityPolicies = append(endPoint.SecurityPolicies, secPol)
}
}
dm.SecurityPoliciesLock.RUnlock()
dm.EndPoints[endPointIdx] = endPoint
}
dm.EndPointsLock.Unlock()
}
} else if dm.Containers[containerID].PidNS == 0 && dm.Containers[containerID].MntNS == 0 {
// this entry was updated by kubernetes before docker detects it
// thus, we here use the info given by kubernetes instead of the info given by docker
container.NamespaceName = dm.Containers[containerID].NamespaceName
container.EndPointName = dm.Containers[containerID].EndPointName
container.Labels = dm.Containers[containerID].Labels
container.ContainerName = dm.Containers[containerID].ContainerName
container.ContainerImage = dm.Containers[containerID].ContainerImage
container.PolicyEnabled = dm.Containers[containerID].PolicyEnabled
container.ProcessVisibilityEnabled = dm.Containers[containerID].ProcessVisibilityEnabled
container.FileVisibilityEnabled = dm.Containers[containerID].FileVisibilityEnabled
container.NetworkVisibilityEnabled = dm.Containers[containerID].NetworkVisibilityEnabled
container.CapabilitiesVisibilityEnabled = dm.Containers[containerID].CapabilitiesVisibilityEnabled
dm.Containers[containerID] = container
dm.ContainersLock.Unlock()
dm.EndPointsLock.Lock()
for idx, endPoint := range dm.EndPoints {
if endPoint.NamespaceName == container.NamespaceName && endPoint.EndPointName == container.EndPointName && kl.ContainsElement(endPoint.Containers, container.ContainerID) {
// update apparmor profiles
if !kl.ContainsElement(endPoint.AppArmorProfiles, container.AppArmorProfile) {
dm.EndPoints[idx].AppArmorProfiles = append(dm.EndPoints[idx].AppArmorProfiles, container.AppArmorProfile)
}
if container.Privileged && dm.EndPoints[idx].PrivilegedContainers != nil {
dm.EndPoints[idx].PrivilegedContainers[container.ContainerName] = struct{}{}
}
endPoint = dm.EndPoints[idx]
break
}
}
dm.EndPointsLock.Unlock()
} else {
dm.ContainersLock.Unlock()
return
}
if !dm.K8sEnabled {
dm.ContainersLock.Lock()
dm.SetContainerVisibility(containerID)
container = dm.Containers[containerID]
dm.ContainersLock.Unlock()
}
if dm.SystemMonitor != nil && cfg.GlobalCfg.Policy {
// for throttling
dm.SystemMonitor.Logger.ContainerNsKey[containerID] = common.OuterKey{
MntNs: container.MntNS,
PidNs: container.PidNS,
}
// update NsMap
dm.SystemMonitor.AddContainerIDToNsMap(containerID, container.NamespaceName, container.PidNS, container.MntNS)
dm.RuntimeEnforcer.RegisterContainer(containerID, container.PidNS, container.MntNS)
if dm.Presets != nil {
dm.Presets.RegisterContainer(containerID, container.PidNS, container.MntNS)
}
if len(endPoint.SecurityPolicies) > 0 { // struct can be empty or no policies registered for the endpoint yet
dm.Logger.UpdateSecurityPolicies("ADDED", endPoint)
if dm.RuntimeEnforcer != nil && endPoint.PolicyEnabled == tp.KubeArmorPolicyEnabled {
// enforce security policies
dm.RuntimeEnforcer.UpdateSecurityPolicies(endPoint)
}
if dm.Presets != nil && endPoint.PolicyEnabled == tp.KubeArmorPolicyEnabled {
// enforce preset rules
dm.Presets.UpdateSecurityPolicies(endPoint)
}
}
}
if cfg.GlobalCfg.StateAgent {
container.Status = "running"
go dm.StateAgent.PushContainerEvent(container, state.EventAdded)
}
dm.Logger.Printf("Detected a container (added/%.12s)", containerID)
} else if action == "stop" || action == "destroy" {
// case 1: kill -> die -> stop
// case 2: kill -> die -> destroy
// case 3: destroy
if !dm.K8sEnabled {
dm.ContainersLock.Lock()
dm.EndPointsLock.Lock()
dm.MatchandRemoveContainerFromEndpoint(containerID)
dm.EndPointsLock.Unlock()
dm.ContainersLock.Unlock()
}
dm.ContainersLock.Lock()
container, ok := dm.Containers[containerID]
if !ok {
dm.ContainersLock.Unlock()
return
}
delete(dm.Containers, containerID)
dm.ContainersLock.Unlock()
dm.EndPointsLock.Lock()
// delete endpoint if no security rules and containers
if !dm.K8sEnabled {
idx := 0
endpointsLength := len(dm.EndPoints)
for idx < endpointsLength {
endpoint := dm.EndPoints[idx]
if container.NamespaceName == endpoint.NamespaceName && container.ContainerName == endpoint.EndPointName &&
len(endpoint.SecurityPolicies) == 0 && len(endpoint.Containers) == 0 {
dm.EndPoints = append(dm.EndPoints[:idx], dm.EndPoints[idx+1:]...)
endpointsLength--
idx--
}
idx++
}
}
for idx, endPoint := range dm.EndPoints {
if endPoint.NamespaceName == container.NamespaceName && endPoint.EndPointName == container.EndPointName && kl.ContainsElement(endPoint.Containers, container.ContainerID) {
// update apparmor profiles
for idxA, profile := range endPoint.AppArmorProfiles {
if profile == container.AppArmorProfile {
dm.EndPoints[idx].AppArmorProfiles = append(dm.EndPoints[idx].AppArmorProfiles[:idxA], dm.EndPoints[idx].AppArmorProfiles[idxA+1:]...)
break
}
}
break
}
}
dm.EndPointsLock.Unlock()
if cfg.GlobalCfg.StateAgent {
container.Status = "terminated"
go dm.StateAgent.PushContainerEvent(container, state.EventDeleted)
}
if dm.SystemMonitor != nil && cfg.GlobalCfg.Policy {
outkey := dm.SystemMonitor.Logger.ContainerNsKey[containerID]
dm.Logger.DeleteAlertMapKey(outkey)
delete(dm.SystemMonitor.Logger.ContainerNsKey, containerID)
// update NsMap
dm.SystemMonitor.DeleteContainerIDFromNsMap(containerID, container.NamespaceName, container.PidNS, container.MntNS)
dm.RuntimeEnforcer.UnregisterContainer(containerID)
if dm.Presets != nil {
dm.Presets.UnregisterContainer(containerID)
}
}
dm.Logger.Printf("Detected a container (removed/%.12s)", containerID)
} else if action == "die" && cfg.GlobalCfg.StateAgent {
// handle die - keep map but update state
dm.ContainersLock.Lock()
container, ok := dm.Containers[containerID]
if !ok {
dm.ContainersLock.Unlock()
return
}
dm.ContainersLock.Unlock()
container.Status = "waiting"
go dm.StateAgent.PushContainerEvent(container, state.EventUpdated)
}
}
// MonitorDockerEvents Function
func (dm *KubeArmorDaemon) MonitorDockerEvents() {
dm.WgDaemon.Add(1)
defer dm.WgDaemon.Done()
// check if Docker exists else instantiate
if Docker == nil {
var err error
Docker, err = NewDockerHandler()
if err != nil {
dm.Logger.Errf("Failed to create new Docker client: %s", err.Error())
return
}
}
dm.Logger.Print("Started to monitor Docker events")
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
EventChan := Docker.GetEventChannel(ctx, StopChan)
for {
select {
case <-StopChan:
return
case msg, valid := <-EventChan:
if !valid {
continue
}
// if message type is container
if msg.Type == "container" {
dm.UpdateDockerContainer(msg.ID, string(msg.Action))
}
}
}
}
// SPDX-License-Identifier: Apache-2.0
// Copyright 2021 Authors of KubeArmor
package core
import (
"bytes"
"context"
"crypto/tls"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"time"
kl "github.com/kubearmor/KubeArmor/KubeArmor/common"
kg "github.com/kubearmor/KubeArmor/KubeArmor/log"
kspclient "github.com/kubearmor/KubeArmor/pkg/KubeArmorController/client/clientset/versioned"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
ctrl "sigs.k8s.io/controller-runtime"
)
// ================= //
// == K8s Handler == //
// ================= //
// K8s Handler
var K8s *K8sHandler
// init Function
func init() {
K8s = NewK8sHandler()
}
// K8sHandler Structure
type K8sHandler struct {
K8sClient *kubernetes.Clientset
KSPClient *kspclient.Clientset
HTTPClient *http.Client
WatchClient *http.Client
K8sToken string
K8sHost string
K8sPort string
}
// NewK8sHandler Function
func NewK8sHandler() *K8sHandler {
kh := &K8sHandler{}
if val, ok := os.LookupEnv("KUBERNETES_SERVICE_HOST"); ok {
kh.K8sHost = val
} else {
kh.K8sHost = "127.0.0.1"
}
if val, ok := os.LookupEnv("KUBERNETES_PORT_443_TCP_PORT"); ok {
kh.K8sPort = val
} else {
kh.K8sPort = "8001" // kube-proxy
}
kh.HTTPClient = &http.Client{
Timeout: time.Second * 5,
// #nosec
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
},
}
kh.WatchClient = &http.Client{
// #nosec
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
},
}
config, err := ctrl.GetConfig()
if err != nil {
kg.Warnf("Error creating kubernetes config, %s", err)
return kh
}
kh.KSPClient, err = kspclient.NewForConfig(config)
if err != nil {
kg.Warnf("Error creating ksp clientset, %s", err)
return kh
}
return kh
}
// ================ //
// == K8s Client == //
// ================ //
// InitK8sClient Function
func (kh *K8sHandler) InitK8sClient() bool {
if !kl.IsK8sEnv() { // not Kubernetes
return false
}
if kh.K8sClient == nil {
if kl.IsInK8sCluster() {
return kh.InitInclusterAPIClient()
}
if kl.IsK8sLocal() {
return kh.InitLocalAPIClient()
}
return false
}
return true
}
// InitLocalAPIClient Function
func (kh *K8sHandler) InitLocalAPIClient() bool {
kubeconfig := os.Getenv("KUBECONFIG")
if kubeconfig == "" {
kubeconfig = os.Getenv("HOME") + "/.kube/config"
if _, err := os.Stat(filepath.Clean(kubeconfig)); err != nil {
return false
}
}
// use the current context in kubeconfig
config, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
if err != nil {
return false
}
// creates the clientset
client, err := kubernetes.NewForConfig(config)
if err != nil {
return false
}
kh.K8sClient = client
return true
}
// InitInclusterAPIClient Function
func (kh *K8sHandler) InitInclusterAPIClient() bool {
read, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/token")
if err != nil {
return false
}
kh.K8sToken = string(read)
// create the configuration by token
kubeConfig := &rest.Config{
Host: "https://" + kh.K8sHost + ":" + kh.K8sPort,
BearerToken: kh.K8sToken,
// #nosec
TLSClientConfig: rest.TLSClientConfig{
Insecure: true,
},
}
client, err := kubernetes.NewForConfig(kubeConfig)
if err != nil {
return false
}
kh.K8sClient = client
return true
}
// ============== //
// == API Call == //
// ============== //
// DoRequest Function
func (kh *K8sHandler) DoRequest(cmd string, data interface{}, path string) ([]byte, error) {
URL := ""
if kl.IsInK8sCluster() {
URL = "https://" + kh.K8sHost + ":" + kh.K8sPort
} else {
URL = "http://" + kh.K8sHost + ":" + kh.K8sPort
}
pbytes, err := json.Marshal(data)
if err != nil {
return nil, err
}
req, err := http.NewRequest(cmd, URL+path, bytes.NewBuffer(pbytes))
if err != nil {
return nil, err
}
if kl.IsInK8sCluster() {
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", kh.K8sToken))
}
resp, err := kh.HTTPClient.Do(req)
if err != nil {
return nil, err
}
resBody, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
}
if err := resp.Body.Close(); err != nil {
kg.Err(err.Error())
}
return resBody, nil
}
// ================ //
// == Deployment == //
// ================ //
// PatchDeploymentWithAppArmorAnnotations Function
func (kh *K8sHandler) PatchResourceWithAppArmorAnnotations(namespaceName, deploymentName string, appArmorAnnotations map[string]string, kind string) error {
if !kl.IsK8sEnv() { // not Kubernetes
return nil
}
spec := `{"spec":{"template":{"metadata":{"annotations":{"kubearmor-policy":"enabled",`
if kind == "CronJob" {
spec = `{"spec":{"jobTemplate":{"spec":{"template":{"metadata":{"annotations":{"kubearmor-policy":"enabled",`
}
count := len(appArmorAnnotations)
for k, v := range appArmorAnnotations {
if v == "unconfined" {
continue
}
spec = spec + `"container.apparmor.security.beta.kubernetes.io/` + k + `":"localhost/` + v + `"`
if count > 1 {
spec = spec + ","
}
count--
}
if kind == "CronJob" {
spec = spec + `}}}}}}}`
} else {
spec = spec + `}}}}}`
}
if kind == "StatefulSet" {
_, err := kh.K8sClient.AppsV1().StatefulSets(namespaceName).Patch(context.Background(), deploymentName, types.StrategicMergePatchType, []byte(spec), metav1.PatchOptions{})
if err != nil {
return err
}
return nil
} else if kind == "ReplicaSet" {
rs, err := kh.K8sClient.AppsV1().ReplicaSets(namespaceName).Get(context.Background(), deploymentName, metav1.GetOptions{})
if err != nil {
return err
}
replicas := *rs.Spec.Replicas
_, err = kh.K8sClient.AppsV1().ReplicaSets(namespaceName).Patch(context.Background(), deploymentName, types.MergePatchType, []byte(spec), metav1.PatchOptions{})
if err != nil {
return err
}
// To update the annotations we need to restart the replicaset,we scale it down and scale it back up
patchData := fmt.Appendf(nil, `{"spec": {"replicas": 0}}`)
_, err = kh.K8sClient.AppsV1().ReplicaSets(namespaceName).Patch(context.Background(), deploymentName, types.StrategicMergePatchType, patchData, metav1.PatchOptions{})
if err != nil {
return err
}
time.Sleep(2 * time.Second)
patchData2 := fmt.Appendf(nil, `{"spec": {"replicas": %d}}`, replicas)
_, err = kh.K8sClient.AppsV1().ReplicaSets(namespaceName).Patch(context.Background(), deploymentName, types.StrategicMergePatchType, patchData2, metav1.PatchOptions{})
if err != nil {
return err
}
return nil
} else if kind == "DaemonSet" {
_, err := kh.K8sClient.AppsV1().DaemonSets(namespaceName).Patch(context.Background(), deploymentName, types.MergePatchType, []byte(spec), metav1.PatchOptions{})
if err != nil {
return err
}
return nil
} else if kind == "Deployment" {
_, err := kh.K8sClient.AppsV1().Deployments(namespaceName).Patch(context.Background(), deploymentName, types.StrategicMergePatchType, []byte(spec), metav1.PatchOptions{})
if err != nil {
return err
}
} else if kind == "CronJob" {
_, err := kh.K8sClient.BatchV1().CronJobs(namespaceName).Patch(context.Background(), deploymentName, types.StrategicMergePatchType, []byte(spec), metav1.PatchOptions{})
if err != nil {
return err
}
} else if kind == "Pod" {
// this condition wont be triggered, handled by controller
return nil
}
return nil
}
// PatchDeploymentWithSELinuxAnnotations Function
func (kh *K8sHandler) PatchDeploymentWithSELinuxAnnotations(namespaceName, deploymentName string, seLinuxAnnotations map[string]string) error {
if !kl.IsK8sEnv() { // not Kubernetes
return nil
}
spec := `{"spec":{"template":{"metadata":{"annotations":{"kubearmor-policy":"enabled",`
count := len(seLinuxAnnotations)
for k, v := range seLinuxAnnotations {
spec = spec + `"kubearmor-selinux/` + k + `":"` + v + `"`
if count > 1 {
spec = spec + ","
}
count--
}
spec = spec + `}}}}}`
_, err := kh.K8sClient.AppsV1().Deployments(namespaceName).Patch(context.Background(), deploymentName, types.StrategicMergePatchType, []byte(spec), metav1.PatchOptions{})
if err != nil {
return err
}
return nil
}
// ================ //
// == ReplicaSet == //
// ================ //
// GetDeploymentNameControllingReplicaSet Function
func (kh *K8sHandler) GetDeploymentNameControllingReplicaSet(namespaceName, podownerName string) (string, string) {
if !kl.IsK8sEnv() { // not Kubernetes
return "", ""
}
// get replicaSet from k8s api client
rs, err := kh.K8sClient.AppsV1().ReplicaSets(namespaceName).Get(context.Background(), podownerName, metav1.GetOptions{})
if err != nil {
return "", ""
}
// check if we have ownerReferences
if len(rs.ObjectMeta.OwnerReferences) == 0 {
return "", ""
}
// check if given ownerReferences are for Deployment
if rs.ObjectMeta.OwnerReferences[0].Kind != "Deployment" {
return "", ""
}
// return the deployment name
return rs.ObjectMeta.OwnerReferences[0].Name, rs.ObjectMeta.Namespace
}
// GetReplicaSet Function
func (kh *K8sHandler) GetReplicaSet(namespaceName, podownerName string) (string, string) {
if !kl.IsK8sEnv() { // not Kubernetes
return "", ""
}
// get replicaSet from k8s api client
rs, err := kh.K8sClient.AppsV1().ReplicaSets(namespaceName).Get(context.Background(), podownerName, metav1.GetOptions{})
if err != nil {
return "", ""
}
// return the replicaSet name
return rs.ObjectMeta.Name, rs.ObjectMeta.Namespace
}
// ================ //
// == DaemonSet == //
// ================ //
// GetDaemonSet Function
func (kh *K8sHandler) GetDaemonSet(namespaceName, podownerName string) (string, string) {
if !kl.IsK8sEnv() { // not Kubernetes
return "", ""
}
// get daemonSet from k8s api client
ds, err := kh.K8sClient.AppsV1().DaemonSets(namespaceName).Get(context.Background(), podownerName, metav1.GetOptions{})
if err != nil {
return "", ""
}
// return the daemonSet name
return ds.ObjectMeta.Name, ds.ObjectMeta.Namespace
}
// ================ //
// == StatefulSet == //
// ================ //
// GetStatefulSet Function
func (kh *K8sHandler) GetStatefulSet(namespaceName, podownerName string) (string, string) {
if !kl.IsK8sEnv() { // not Kubernetes
return "", ""
}
// get statefulSets from k8s api client
ss, err := kh.K8sClient.AppsV1().StatefulSets(namespaceName).Get(context.Background(), podownerName, metav1.GetOptions{})
if err != nil {
return "", ""
}
// return the statefulSet name
return ss.ObjectMeta.Name, ss.ObjectMeta.Namespace
}
// ====================== //
// == Custom Resources == //
// ====================== //
// CheckCustomResourceDefinition Function
func (kh *K8sHandler) CheckCustomResourceDefinition(resourceName string) bool {
if !kl.IsK8sEnv() { // not Kubernetes
return false
}
exist := false
apiGroup := metav1.APIGroup{}
// check APIGroup
if resBody, errOut := kh.DoRequest("GET", nil, "/apis"); errOut == nil {
res := metav1.APIGroupList{}
if errIn := json.Unmarshal(resBody, &res); errIn == nil {
for _, group := range res.Groups {
if group.Name == "security.kubearmor.com" {
exist = true
apiGroup = group
break
}
}
}
}
// check APIResource
if exist {
if resBody, errOut := kh.DoRequest("GET", nil, "/apis/"+apiGroup.PreferredVersion.GroupVersion); errOut == nil {
res := metav1.APIResourceList{}
if errIn := json.Unmarshal(resBody, &res); errIn == nil {
for _, resource := range res.APIResources {
if resource.Name == resourceName {
return true
}
}
}
}
}
return false
}
// WatchK8sSecurityPolicies Function
func (kh *K8sHandler) WatchK8sSecurityPolicies() *http.Response {
if !kl.IsK8sEnv() { // not Kubernetes
return nil
}
if kl.IsInK8sCluster() {
URL := "https://" + kh.K8sHost + ":" + kh.K8sPort + "/apis/security.kubearmor.com/v1/kubearmorpolicies?watch=true"
req, err := http.NewRequest("GET", URL, nil)
if err != nil {
return nil
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", kh.K8sToken))
resp, err := kh.WatchClient.Do(req)
if err != nil {
return nil
}
return resp
}
// kube-proxy (local)
URL := "http://" + kh.K8sHost + ":" + kh.K8sPort + "/apis/security.kubearmor.com/v1/kubearmorpolicies?watch=true"
if resp, err := http.Get(URL); err == nil /* #nosec */ {
return resp
}
return nil
}
// WatchK8sHostSecurityPolicies Function
func (kh *K8sHandler) WatchK8sHostSecurityPolicies() *http.Response {
if !kl.IsK8sEnv() { // not Kubernetes
return nil
}
if kl.IsInK8sCluster() {
URL := "https://" + kh.K8sHost + ":" + kh.K8sPort + "/apis/security.kubearmor.com/v1/kubearmorhostpolicies?watch=true"
req, err := http.NewRequest("GET", URL, nil)
if err != nil {
return nil
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", kh.K8sToken))
resp, err := kh.WatchClient.Do(req)
if err != nil {
return nil
}
return resp
}
// kube-proxy (local)
URL := "http://" + kh.K8sHost + ":" + kh.K8sPort + "/apis/security.kubearmor.com/v1/kubearmorhostpolicies?watch=true"
if resp, err := http.Get(URL); err == nil /* #nosec */ {
return resp
}
return nil
}
// this function get the owner details of a pod
func getTopLevelOwner(obj metav1.ObjectMeta, namespace string, objkind string) (string, string, string, error) {
ownerRef := kl.GetControllingPodOwner(obj.OwnerReferences)
if ownerRef == nil {
return obj.Name, objkind, namespace, nil
}
switch ownerRef.Kind {
case "Pod":
pod, err := K8s.K8sClient.CoreV1().Pods(namespace).Get(context.Background(), ownerRef.Name, metav1.GetOptions{})
if err != nil {
return "", "", "", err
}
if len(pod.OwnerReferences) > 0 {
return getTopLevelOwner(pod.ObjectMeta, namespace, "Pod")
}
case "Job":
job, err := K8s.K8sClient.BatchV1().Jobs(namespace).Get(context.Background(), ownerRef.Name, metav1.GetOptions{})
if err != nil {
return "", "", "", err
}
if len(job.OwnerReferences) > 0 {
return getTopLevelOwner(job.ObjectMeta, namespace, "CronJob")
}
return job.Name, "Job", job.Namespace, nil
case "CronJob":
cronJob, err := K8s.K8sClient.BatchV1().CronJobs(namespace).Get(context.Background(), ownerRef.Name, metav1.GetOptions{})
if err != nil {
return "", "", "", err
}
if len(cronJob.OwnerReferences) > 0 {
return getTopLevelOwner(cronJob.ObjectMeta, namespace, "CronJob")
}
return cronJob.Name, "CronJob", cronJob.Namespace, nil
case "Deployment":
deployment, err := K8s.K8sClient.AppsV1().Deployments(namespace).Get(context.Background(), ownerRef.Name, metav1.GetOptions{})
if err != nil {
return "", "", "", err
}
if len(deployment.OwnerReferences) > 0 {
return getTopLevelOwner(deployment.ObjectMeta, namespace, "Deployment")
}
return deployment.Name, "Deployment", deployment.Namespace, nil
case "ReplicaSet":
replicaset, err := K8s.K8sClient.AppsV1().ReplicaSets(namespace).Get(context.Background(), ownerRef.Name, metav1.GetOptions{})
if err != nil {
return "", "", "", err
}
if len(replicaset.OwnerReferences) > 0 {
return getTopLevelOwner(replicaset.ObjectMeta, namespace, "ReplicaSet")
}
return replicaset.Name, "ReplicaSet", replicaset.Namespace, nil
case "StatefulSet":
statefulset, err := K8s.K8sClient.AppsV1().StatefulSets(namespace).Get(context.Background(), ownerRef.Name, metav1.GetOptions{})
if err != nil {
return "", "", "", err
}
if len(statefulset.OwnerReferences) > 0 {
return getTopLevelOwner(statefulset.ObjectMeta, namespace, "StatefulSet")
}
return statefulset.Name, "StatefulSet", statefulset.Namespace, nil
case "DaemonSet":
daemonset, err := K8s.K8sClient.AppsV1().DaemonSets(namespace).Get(context.Background(), ownerRef.Name, metav1.GetOptions{})
if err != nil {
return "", "", "", err
}
if len(daemonset.OwnerReferences) > 0 {
return getTopLevelOwner(daemonset.ObjectMeta, namespace, "DaemonSet")
}
return daemonset.Name, "DaemonSet", daemonset.Namespace, nil
// Default case when
default:
return obj.Name, objkind, namespace, nil
}
return "", "", "", nil
}
// SPDX-License-Identifier: Apache-2.0
// Copyright 2022 Authors of KubeArmor
package core
import (
"context"
"encoding/json"
kl "github.com/kubearmor/KubeArmor/KubeArmor/common"
cfg "github.com/kubearmor/KubeArmor/KubeArmor/config"
tp "github.com/kubearmor/KubeArmor/KubeArmor/types"
pb "github.com/kubearmor/KubeArmor/protobuf"
"google.golang.org/protobuf/types/known/emptypb"
)
// KarmorData Structure
type KarmorData struct {
OSImage string
KernelVersion string
KubeletVersion string
ContainerRuntime string
ActiveLSM string
KernelHeaderPresent bool
HostSecurity bool
ContainerSecurity bool
ContainerDefaultPosture tp.DefaultPosture
HostDefaultPosture tp.DefaultPosture
HostVisibility string
}
// Probe provides structure to serve Policy gRPC service
type Probe struct {
pb.ProbeServiceServer
GetContainerData func() ([]string, map[string]*pb.ContainerData, map[string]*pb.HostSecurityPolicies)
}
// SetKarmorData generates runtime configuration for KubeArmor to be consumed by kArmor
func (dm *KubeArmorDaemon) SetKarmorData() {
var kd KarmorData
kd.ContainerDefaultPosture = tp.DefaultPosture{
FileAction: cfg.GlobalCfg.DefaultFilePosture,
NetworkAction: cfg.GlobalCfg.DefaultNetworkPosture,
CapabilitiesAction: cfg.GlobalCfg.DefaultCapabilitiesPosture,
}
kd.HostDefaultPosture = tp.DefaultPosture{
FileAction: cfg.GlobalCfg.HostDefaultFilePosture,
NetworkAction: cfg.GlobalCfg.HostDefaultNetworkPosture,
CapabilitiesAction: cfg.GlobalCfg.HostDefaultCapabilitiesPosture,
}
kd.OSImage = dm.Node.OSImage
kd.ContainerRuntime = dm.Node.ContainerRuntimeVersion
kd.KernelVersion = dm.Node.KernelVersion
kd.KubeletVersion = dm.Node.KubeletVersion
kd.ContainerRuntime = dm.Node.ContainerRuntimeVersion
if dm.RuntimeEnforcer != nil {
kd.ActiveLSM = dm.RuntimeEnforcer.EnforcerType
if cfg.GlobalCfg.Policy {
kd.ContainerSecurity = true
}
if cfg.GlobalCfg.HostPolicy {
kd.HostSecurity = true
}
}
kd.KernelHeaderPresent = true //this is always true since KubeArmor is running
kd.HostVisibility = dm.Node.Annotations["kubearmor-visibility"]
err := kl.WriteToFile(kd, "/tmp/karmorProbeData.cfg")
if err != nil {
dm.Logger.Errf("Error writing karmor config data (%s)", err.Error())
}
}
// SetProbeContainerData keeps track of containers and the applied policies
func (dm *KubeArmorDaemon) SetProbeContainerData() ([]string, map[string]*pb.ContainerData, map[string]*pb.HostSecurityPolicies) {
var containerlist []string
dm.ContainersLock.Lock()
for _, value := range dm.Containers {
containerlist = append(containerlist, value.ContainerName)
}
dm.ContainersLock.Unlock()
containerMap := make(map[string]*pb.ContainerData)
dm.EndPointsLock.Lock()
for _, ep := range dm.EndPoints {
var policyNames []string
var policyData []*pb.Policy
for _, policy := range ep.SecurityPolicies {
policyNames = append(policyNames, policy.Metadata["policyName"])
policyEventData, err := json.Marshal(policy)
if err != nil {
dm.Logger.Errf("Error marshalling policy data (%s)", err.Error())
} else {
policyData = append(policyData, &pb.Policy{Policy: policyEventData})
}
}
containerMap[ep.EndPointName] = &pb.ContainerData{
PolicyList: policyNames,
PolicyEnabled: int32(ep.PolicyEnabled),
PolicyDataList: policyData,
}
}
dm.EndPointsLock.Unlock()
// Mapping HostPolicies to their host hostName : HostPolicy
hostMap := make(map[string]*pb.HostSecurityPolicies)
dm.HostSecurityPoliciesLock.Lock()
for _, hp := range dm.HostSecurityPolicies {
hostName := dm.Node.NodeName
if val, ok := hostMap[hostName]; ok {
val.PolicyList = append(val.PolicyList, hp.Metadata["policyName"])
policyEventData, err := json.Marshal(hp)
if err != nil {
dm.Logger.Errf("Error marshalling policy data (%s)", err.Error())
} else {
val.PolicyDataList = append(val.PolicyDataList, &pb.Policy{
Policy: policyEventData,
})
}
hostMap[hostName] = val
} else {
policyEventData, err := json.Marshal(hp)
if err != nil {
dm.Logger.Errf("Error marshalling policy data (%s)", err.Error())
}
hostMap[hostName] = &pb.HostSecurityPolicies{
PolicyList: []string{hp.Metadata["policyName"]},
PolicyDataList: []*pb.Policy{{Policy: policyEventData}},
}
}
}
dm.HostSecurityPoliciesLock.Unlock()
return containerlist, containerMap, hostMap
}
// GetProbeData sends policy data through grpc client
func (p *Probe) GetProbeData(c context.Context, in *emptypb.Empty) (*pb.ProbeResponse, error) {
containerList, containerMap, hostMap := p.GetContainerData()
res := &pb.ProbeResponse{
ContainerList: containerList,
ContainerMap: containerMap,
HostMap: hostMap,
}
return res, nil
}
// SPDX-License-Identifier: Apache-2.0
// Copyright 2022 Authors of KubeArmor
// Package core is responsible for initiating and maintaining interactions between external entities like K8s,CRIs and internal KubeArmor entities like eBPF Monitor and Log Feeders
package core
import (
"context"
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"os"
"os/signal"
"strings"
"sync"
"syscall"
"time"
"github.com/kubearmor/KubeArmor/KubeArmor/common"
kl "github.com/kubearmor/KubeArmor/KubeArmor/common"
cfg "github.com/kubearmor/KubeArmor/KubeArmor/config"
kg "github.com/kubearmor/KubeArmor/KubeArmor/log"
"github.com/kubearmor/KubeArmor/KubeArmor/policy"
"github.com/kubearmor/KubeArmor/KubeArmor/presets"
"github.com/kubearmor/KubeArmor/KubeArmor/state"
tp "github.com/kubearmor/KubeArmor/KubeArmor/types"
"google.golang.org/grpc/health"
"google.golang.org/grpc/health/grpc_health_v1"
"google.golang.org/grpc/reflection"
"k8s.io/client-go/tools/cache"
efc "github.com/kubearmor/KubeArmor/KubeArmor/enforcer"
fd "github.com/kubearmor/KubeArmor/KubeArmor/feeder"
kvm "github.com/kubearmor/KubeArmor/KubeArmor/kvmAgent"
mon "github.com/kubearmor/KubeArmor/KubeArmor/monitor"
pb "github.com/kubearmor/KubeArmor/protobuf"
)
// ====================== //
// == KubeArmor Daemon == //
// ====================== //
// StopChan Channel
var StopChan chan struct{}
// init Function
func init() {
StopChan = make(chan struct{})
}
// KubeArmorDaemon Structure
type KubeArmorDaemon struct {
// node
Node tp.Node
NodeLock *sync.RWMutex
// flag
K8sEnabled bool
// K8s pods (from kubernetes)
K8sPods []tp.K8sPod
K8sPodsLock *sync.RWMutex
// containers (from docker)
Containers map[string]tp.Container
ContainersLock *sync.RWMutex
// endpoints
EndPoints []tp.EndPoint
EndPointsLock *sync.RWMutex
// Owner Info
OwnerInfo map[string]tp.PodOwner
// Security policies
SecurityPolicies []tp.SecurityPolicy
SecurityPoliciesLock *sync.RWMutex
// Host Security policies
HostSecurityPolicies []tp.HostSecurityPolicy
HostSecurityPoliciesLock *sync.RWMutex
//DefaultPosture (namespace -> postures)
DefaultPostures map[string]tp.DefaultPosture
DefaultPosturesLock *sync.Mutex
// pid map
ActiveHostPidMap map[string]tp.PidMap
ActivePidMapLock *sync.RWMutex
// logger
Logger *fd.Feeder
// system monitor
SystemMonitor *mon.SystemMonitor
// runtime enforcer
RuntimeEnforcer *efc.RuntimeEnforcer
// presets
Presets *presets.Preset
// kvm agent
KVMAgent *kvm.KVMAgent
// state agent
StateAgent *state.StateAgent
// WgDaemon Handler
WgDaemon sync.WaitGroup
// system monitor lock
MonitorLock *sync.RWMutex
// health-server
GRPCHealthServer *health.Server
}
// NewKubeArmorDaemon Function
func NewKubeArmorDaemon() *KubeArmorDaemon {
dm := new(KubeArmorDaemon)
dm.Node = tp.Node{}
dm.NodeLock = new(sync.RWMutex)
dm.K8sEnabled = false
dm.K8sPods = []tp.K8sPod{}
dm.K8sPodsLock = new(sync.RWMutex)
dm.Containers = map[string]tp.Container{}
dm.ContainersLock = new(sync.RWMutex)
dm.EndPoints = []tp.EndPoint{}
dm.EndPointsLock = new(sync.RWMutex)
dm.SecurityPolicies = []tp.SecurityPolicy{}
dm.SecurityPoliciesLock = new(sync.RWMutex)
dm.HostSecurityPolicies = []tp.HostSecurityPolicy{}
dm.HostSecurityPoliciesLock = new(sync.RWMutex)
dm.DefaultPostures = map[string]tp.DefaultPosture{}
dm.DefaultPosturesLock = new(sync.Mutex)
dm.ActiveHostPidMap = map[string]tp.PidMap{}
dm.ActivePidMapLock = new(sync.RWMutex)
dm.Logger = nil
dm.SystemMonitor = nil
dm.RuntimeEnforcer = nil
dm.KVMAgent = nil
dm.WgDaemon = sync.WaitGroup{}
dm.MonitorLock = new(sync.RWMutex)
dm.OwnerInfo = map[string]tp.PodOwner{}
return dm
}
// DestroyKubeArmorDaemon Function
func (dm *KubeArmorDaemon) DestroyKubeArmorDaemon() {
close(StopChan)
if dm.SystemMonitor != nil {
// close system monitor
if dm.CloseSystemMonitor() {
dm.Logger.Print("Stopped KubeArmor Monitor")
}
}
if dm.RuntimeEnforcer != nil {
// close runtime enforcer
if dm.CloseRuntimeEnforcer() {
dm.Logger.Print("Stopped KubeArmor Enforcer")
}
}
if dm.KVMAgent != nil {
// close kvm agent
if dm.CloseKVMAgent() {
dm.Logger.Print("Stopped KVM Agent")
}
}
if dm.Logger != nil {
dm.Logger.Print("Terminated KubeArmor")
} else {
kg.Print("Terminated KubeArmor")
}
if dm.StateAgent != nil {
//go dm.StateAgent.PushNodeEvent(dm.Node, state.EventDeleted)
if dm.CloseStateAgent() {
kg.Print("Destroyed StateAgent")
}
}
// wait for a while
time.Sleep(time.Second * 1)
if dm.Logger != nil {
// close logger
if dm.CloseLogger() {
kg.Print("Stopped KubeArmor Logger")
}
}
// wait for other routines
kg.Print("Waiting for routine terminations")
dm.WgDaemon.Wait()
// delete pid file
if _, err := os.Stat(cfg.PIDFilePath); err == nil {
kg.Print("Deleting PID file")
err := os.Remove(cfg.PIDFilePath)
if err != nil {
kg.Errf("Failed to delete PID file")
}
}
}
// ============ //
// == Logger == //
// ============ //
// InitLogger Function
func (dm *KubeArmorDaemon) InitLogger() bool {
dm.Logger = fd.NewFeeder(&dm.Node, &dm.NodeLock)
return dm.Logger != nil
}
// ServeLogFeeds Function
func (dm *KubeArmorDaemon) ServeLogFeeds() {
dm.WgDaemon.Add(1)
defer dm.WgDaemon.Done()
go dm.Logger.ServeLogFeeds()
}
// CloseLogger Function
func (dm *KubeArmorDaemon) CloseLogger() bool {
if err := dm.Logger.DestroyFeeder(); err != nil {
kg.Errf("Failed to destroy KubeArmor Logger (%s)", err.Error())
return false
}
return true
}
// ==================== //
// == System Monitor == //
// ==================== //
// InitSystemMonitor Function
func (dm *KubeArmorDaemon) InitSystemMonitor() bool {
dm.SystemMonitor = mon.NewSystemMonitor(&dm.Node, &dm.NodeLock, dm.Logger, &dm.Containers, &dm.ContainersLock, &dm.ActiveHostPidMap, &dm.ActivePidMapLock, &dm.MonitorLock)
if dm.SystemMonitor == nil {
return false
}
if err := dm.SystemMonitor.InitBPF(); err != nil {
kg.Errf("Failed to initialize BPF (%s)", err.Error())
return false
}
return true
}
// MonitorSystemEvents Function
func (dm *KubeArmorDaemon) MonitorSystemEvents() {
dm.WgDaemon.Add(1)
defer dm.WgDaemon.Done()
if cfg.GlobalCfg.Policy || cfg.GlobalCfg.HostPolicy {
go dm.SystemMonitor.TraceSyscall()
go dm.SystemMonitor.UpdateLogs()
go dm.SystemMonitor.CleanUpExitedHostPids()
}
}
// CloseSystemMonitor Function
func (dm *KubeArmorDaemon) CloseSystemMonitor() bool {
if err := dm.SystemMonitor.DestroySystemMonitor(); err != nil {
dm.Logger.Errf("Failed to destroy KubeArmor Monitor (%s)", err.Error())
return false
}
return true
}
// ====================== //
// == Runtime Enforcer == //
// ====================== //
// InitRuntimeEnforcer Function
func (dm *KubeArmorDaemon) InitRuntimeEnforcer(pinpath string) bool {
dm.RuntimeEnforcer = efc.NewRuntimeEnforcer(dm.Node, pinpath, dm.Logger, dm.SystemMonitor)
return dm.RuntimeEnforcer != nil
}
// CloseRuntimeEnforcer Function
func (dm *KubeArmorDaemon) CloseRuntimeEnforcer() bool {
if err := dm.RuntimeEnforcer.DestroyRuntimeEnforcer(); err != nil {
dm.Logger.Errf("Failed to destory KubeArmor Enforcer (%s)", err.Error())
return false
}
return true
}
// ============= //
// == Presets == //
// ============= //
// InitPresets Function
func (dm *KubeArmorDaemon) InitPresets(logger *fd.Feeder, monitor *mon.SystemMonitor) bool {
dm.Presets = presets.NewPreset(dm.Logger, dm.SystemMonitor)
return dm.Presets != nil
}
// ClosePresets Function
func (dm *KubeArmorDaemon) ClosePresets() bool {
if err := dm.Presets.Destroy(); err != nil {
dm.Logger.Errf("Failed to destroy preset (%s)", err.Error())
return false
}
return true
}
// =============== //
// == KVM Agent == //
// =============== //
// InitKVMAgent Function
func (dm *KubeArmorDaemon) InitKVMAgent() bool {
dm.KVMAgent = kvm.NewKVMAgent(dm.ParseAndUpdateHostSecurityPolicy)
return dm.KVMAgent != nil
}
// ConnectToKVMService Function
func (dm *KubeArmorDaemon) ConnectToKVMService() {
go dm.KVMAgent.ConnectToKVMService()
}
// CloseKVMAgent Function
func (dm *KubeArmorDaemon) CloseKVMAgent() bool {
if err := dm.KVMAgent.DestroyKVMAgent(); err != nil {
dm.Logger.Errf("Failed to destory KVM Agent (%s)", err.Error())
return false
}
return true
}
// ================= //
// == State Agent == //
// ================= //
// InitStateAgent Function
func (dm *KubeArmorDaemon) InitStateAgent() bool {
dm.StateAgent = state.NewStateAgent(&dm.Node, dm.NodeLock, dm.Containers, dm.ContainersLock)
return dm.StateAgent != nil
}
// CloseStateAgent Function
func (dm *KubeArmorDaemon) CloseStateAgent() bool {
if err := dm.StateAgent.DestroyStateAgent(); err != nil {
dm.Logger.Errf("Failed to destory State Agent (%s)", err.Error())
return false
}
return true
}
// ==================== //
// == Signal Handler == //
// ==================== //
// GetOSSigChannel Function
func GetOSSigChannel() chan os.Signal {
c := make(chan os.Signal, 1)
signal.Notify(c,
syscall.SIGHUP,
syscall.SIGINT,
syscall.SIGTERM,
syscall.SIGQUIT,
os.Interrupt)
return c
}
// =================== //
// == Health Server == //
// =================== //
func (dm *KubeArmorDaemon) SetHealthStatus(serviceName string, healthStatus grpc_health_v1.HealthCheckResponse_ServingStatus) bool {
if dm.GRPCHealthServer != nil {
dm.GRPCHealthServer.SetServingStatus(serviceName, healthStatus)
return true
}
return false
}
// ========== //
// == Main == //
// ========== //
// KubeArmor Function
func KubeArmor() {
// create a daemon
dm := NewKubeArmorDaemon()
// Enable KubeArmorHostPolicy for both VM and KVMAgent and in non-k8s env
if cfg.GlobalCfg.KVMAgent || (!cfg.GlobalCfg.K8sEnv && cfg.GlobalCfg.HostPolicy) {
dm.NodeLock.Lock()
dm.Node.NodeName = cfg.GlobalCfg.Host
dm.Node.NodeIP = kl.GetExternalIPAddr()
// add identity for matching node selector
dm.Node.Labels = make(map[string]string)
dm.Node.Labels["kubearmor.io/hostname"] = dm.Node.NodeName
dm.Node.Identities = append(dm.Node.Identities, "kubearmor.io/hostname"+"="+dm.Node.NodeName)
dm.Node.Annotations = map[string]string{}
dm.HandleNodeAnnotations(&dm.Node)
hostInfo := kl.GetCommandOutputWithoutErr("hostnamectl", []string{})
for line := range strings.SplitSeq(hostInfo, "\n") {
if strings.Contains(line, "Machine ID") {
dm.Node.NodeID = strings.Split(line, ": ")[1]
}
if strings.Contains(line, "Operating System") {
dm.Node.OSImage = strings.Split(line, ": ")[1]
}
}
dm.Node.LastUpdatedAt = kl.GetBootTime()
dm.Node.KernelVersion = kl.GetCommandOutputWithoutErr("uname", []string{"-r"})
dm.Node.KernelVersion = strings.TrimSuffix(dm.Node.KernelVersion, "\n")
dm.WatchConfigChanges()
dm.NodeLock.Unlock()
} else if cfg.GlobalCfg.K8sEnv {
if !K8s.InitK8sClient() {
kg.Err("Failed to initialize Kubernetes client")
// destroy the daemon
dm.DestroyKubeArmorDaemon()
return
}
kg.Print("Initialized Kubernetes client")
// set the flag
dm.K8sEnabled = true
// watch k8s nodes
go dm.WatchK8sNodes()
kg.Print("Started to monitor node events")
// == //
// wait for a while
time.Sleep(time.Second * 1)
for timeout := 0; timeout <= 60; timeout++ {
// read node information
dm.NodeLock.RLock()
nodeIP := dm.Node.NodeIP
dm.NodeLock.RUnlock()
if nodeIP != "" {
break
}
if nodeIP == "" && timeout == 60 {
kg.Print("The node information is not available, terminating KubeArmor")
// destroy the daemon
dm.DestroyKubeArmorDaemon()
return
}
kg.Print("The node information is not available")
// wait for a while
time.Sleep(time.Second * 1)
}
}
protectedID := func(id, key string) string {
mac := hmac.New(sha256.New, []byte(id))
mac.Write([]byte(key))
return hex.EncodeToString(mac.Sum(nil))
}
if dm.Node.NodeID == "" {
id, _ := os.ReadFile(cfg.GlobalCfg.MachineIDPath)
dm.Node.NodeID = strings.TrimSuffix(string(id), "\n")
}
dm.Node.NodeID = protectedID(dm.Node.NodeID, dm.Node.NodeName)
kg.Printf("Node Name: %s", dm.Node.NodeName)
kg.Printf("Node IP: %s", dm.Node.NodeIP)
kg.Printf("Node ID: %s", dm.Node.NodeID)
if dm.K8sEnabled {
kg.Printf("Node Annotations: %v", dm.Node.Annotations)
}
kg.Printf("OS Image: %s", dm.Node.OSImage)
kg.Printf("Kernel Version: %s", dm.Node.KernelVersion)
if dm.K8sEnabled {
kg.Printf("Kubelet Version: %s", dm.Node.KubeletVersion)
kg.Printf("Container Runtime: %s", dm.Node.ContainerRuntimeVersion)
}
// == //
// initialize log feeder
if !dm.InitLogger() {
kg.Err("Failed to initialize KubeArmor Logger")
// destroy the daemon
dm.DestroyKubeArmorDaemon()
return
}
dm.Logger.Print("Initialized KubeArmor Logger")
// == //
// health server
if dm.Logger.LogServer != nil {
dm.GRPCHealthServer = health.NewServer()
grpc_health_v1.RegisterHealthServer(dm.Logger.LogServer, dm.GRPCHealthServer)
}
// Init StateAgent
if !dm.K8sEnabled && cfg.GlobalCfg.StateAgent {
dm.NodeLock.Lock()
dm.Node.ClusterName = cfg.GlobalCfg.Cluster
dm.NodeLock.Unlock()
// initialize state agent
if !dm.InitStateAgent() {
dm.Logger.Err("Failed to initialize State Agent Server")
// destroy the daemon
dm.DestroyKubeArmorDaemon()
return
}
dm.Logger.Print("Initialized State Agent Server")
pb.RegisterStateAgentServer(dm.Logger.LogServer, dm.StateAgent)
dm.SetHealthStatus(pb.StateAgent_ServiceDesc.ServiceName, grpc_health_v1.HealthCheckResponse_SERVING)
}
if dm.StateAgent != nil {
go dm.StateAgent.PushNodeEvent(dm.Node, state.EventAdded)
}
// == //
// Containerized workloads with Host
if cfg.GlobalCfg.Policy || cfg.GlobalCfg.HostPolicy {
// initialize system monitor
if !dm.InitSystemMonitor() {
dm.Logger.Err("Failed to initialize KubeArmor Monitor")
// destroy the daemon
dm.DestroyKubeArmorDaemon()
return
}
dm.Logger.Print("Initialized KubeArmor Monitor")
// monitor system events
go dm.MonitorSystemEvents()
dm.Logger.Print("Started to monitor system events")
// initialize runtime enforcer
if !dm.InitRuntimeEnforcer(dm.SystemMonitor.PinPath) {
dm.Logger.Print("Disabled KubeArmor Enforcer since No LSM is enabled")
} else {
dm.Logger.Print("Initialized KubeArmor Enforcer")
if cfg.GlobalCfg.Policy && !cfg.GlobalCfg.HostPolicy {
dm.Logger.Print("Started to protect containers")
} else if !cfg.GlobalCfg.Policy && cfg.GlobalCfg.HostPolicy {
dm.Logger.Print("Started to protect a host")
} else if cfg.GlobalCfg.Policy && cfg.GlobalCfg.HostPolicy {
dm.Logger.Print("Started to protect a host and containers")
}
}
// initialize presets
if !dm.InitPresets(dm.Logger, dm.SystemMonitor) {
dm.Logger.Print("Disabled Presets since no presets are enabled")
} else {
dm.Logger.Print("Initialized Presets")
}
}
enableContainerPolicy := true
dm.SystemMonitor.Logger.ContainerNsKey = make(map[string]common.OuterKey)
// Un-orchestrated workloads
if !dm.K8sEnabled && cfg.GlobalCfg.Policy {
// Check if cri socket set, if not then auto detect
if cfg.GlobalCfg.CRISocket == "" {
if kl.GetCRISocket("") == "" {
dm.Logger.Warnf("Error while looking for CRI socket file")
enableContainerPolicy = false
} else {
cfg.GlobalCfg.CRISocket = "unix://" + kl.GetCRISocket("")
}
} else {
// CRI socket supplied by user, check for existence
criSocketPath := strings.TrimPrefix(cfg.GlobalCfg.CRISocket, "unix://")
_, err := os.Stat(criSocketPath)
if err != nil {
enableContainerPolicy = false
dm.Logger.Warnf("Error while looking for CRI socket file %s", err.Error())
}
}
if enableContainerPolicy {
dm.SetContainerNSVisibility()
// monitor containers
if strings.Contains(cfg.GlobalCfg.CRISocket, "docker") {
// update already deployed containers
dm.GetAlreadyDeployedDockerContainers()
// monitor docker events
go dm.MonitorDockerEvents()
} else if strings.Contains(cfg.GlobalCfg.CRISocket, "containerd") {
// insuring NRI monitoring only in case containerd is present
if cfg.GlobalCfg.NRIEnabled && dm.checkNRIAvailability() {
// monitor NRI events
go dm.MonitorNRIEvents()
} else {
// monitor containerd events
go dm.MonitorContainerdEvents()
}
} else if strings.Contains(cfg.GlobalCfg.CRISocket, "cri-o") {
// monitor crio events
go dm.MonitorCrioEvents()
} else {
enableContainerPolicy = false
dm.Logger.Warnf("Failed to monitor containers: %s is not a supported CRI socket.", cfg.GlobalCfg.CRISocket)
}
dm.Logger.Printf("Using %s for monitoring containers", cfg.GlobalCfg.CRISocket)
}
}
if dm.K8sEnabled && cfg.GlobalCfg.Policy {
if dm.checkNRIAvailability() {
// monitor NRI events
go dm.MonitorNRIEvents()
} else if cfg.GlobalCfg.CRISocket != "" { // check if the CRI socket set while executing kubearmor exists
trimmedSocket := strings.TrimPrefix(cfg.GlobalCfg.CRISocket, "unix://")
if _, err := os.Stat(trimmedSocket); err != nil {
dm.Logger.Warnf("Error while looking for CRI socket file: %s", err.Error())
// destroy the daemon
dm.DestroyKubeArmorDaemon()
return
}
// monitor containers
if strings.Contains(dm.Node.ContainerRuntimeVersion, "docker") || strings.Contains(cfg.GlobalCfg.CRISocket, "docker") {
// update already deployed containers
dm.GetAlreadyDeployedDockerContainers()
// monitor docker events
go dm.MonitorDockerEvents()
} else if strings.Contains(dm.Node.ContainerRuntimeVersion, "containerd") || strings.Contains(cfg.GlobalCfg.CRISocket, "containerd") {
// monitor containerd events
go dm.MonitorContainerdEvents()
} else if strings.Contains(dm.Node.ContainerRuntimeVersion, "cri-o") || strings.Contains(cfg.GlobalCfg.CRISocket, "cri-o") {
// monitor crio events
go dm.MonitorCrioEvents()
} else {
dm.Logger.Errf("Failed to monitor containers: %s is not a supported CRI socket.", cfg.GlobalCfg.CRISocket)
// destroy the daemon
dm.DestroyKubeArmorDaemon()
return
}
dm.Logger.Printf("Using %s for monitoring containers", cfg.GlobalCfg.CRISocket)
} else { // CRI socket not set, we'll have to auto detect
dm.Logger.Print("CRI socket not set. Trying to detect.")
if strings.HasPrefix(dm.Node.ContainerRuntimeVersion, "docker") {
socketFile := kl.GetCRISocket("docker")
if socketFile != "" {
cfg.GlobalCfg.CRISocket = "unix://" + socketFile
// update already deployed containers
dm.GetAlreadyDeployedDockerContainers()
// monitor docker events
go dm.MonitorDockerEvents()
} else {
// we might have to use containerd's socket as docker's socket is not
// available
socketFile := kl.GetCRISocket("containerd")
if socketFile != "" {
cfg.GlobalCfg.CRISocket = "unix://" + socketFile
// monitor containerd events
go dm.MonitorContainerdEvents()
} else {
dm.Logger.Err("Failed to monitor containers (Docker socket file is not accessible)")
// destroy the daemon
dm.DestroyKubeArmorDaemon()
return
}
}
} else if strings.HasPrefix(dm.Node.ContainerRuntimeVersion, "containerd") { // containerd
socketFile := kl.GetCRISocket("containerd")
if socketFile != "" {
cfg.GlobalCfg.CRISocket = "unix://" + socketFile
// monitor containerd events
go dm.MonitorContainerdEvents()
} else {
dm.Logger.Err("Failed to monitor containers (Containerd socket file is not accessible)")
// destroy the daemon
dm.DestroyKubeArmorDaemon()
return
}
} else if strings.HasPrefix(dm.Node.ContainerRuntimeVersion, "cri-o") { // cri-o
socketFile := kl.GetCRISocket("cri-o")
if socketFile != "" {
cfg.GlobalCfg.CRISocket = "unix://" + socketFile
// monitor cri-o events
go dm.MonitorCrioEvents()
} else {
dm.Logger.Err("Failed to monitor containers (CRI-O socket file is not accessible)")
// destroy the daemon
dm.DestroyKubeArmorDaemon()
return
}
}
}
}
// == //
// wait for a while
time.Sleep(time.Second * 1)
// == //
timeout, err := time.ParseDuration(cfg.GlobalCfg.InitTimeout)
if dm.K8sEnabled && cfg.GlobalCfg.Policy {
if err != nil {
dm.Logger.Warnf("Not a valid InitTimeout duration: %q, defaulting to '60s'", cfg.GlobalCfg.InitTimeout)
timeout = 60 * time.Second
}
// watch security policies
securityPoliciesSynced := dm.WatchSecurityPolicies()
if securityPoliciesSynced == nil {
// destroy the daemon
dm.DestroyKubeArmorDaemon()
return
}
dm.Logger.Print("Started to monitor security policies")
// watch cluster security policies
clusterSecurityPoliciesSynced := dm.WatchClusterSecurityPolicies(timeout)
if clusterSecurityPoliciesSynced == nil {
dm.Logger.Warn("error while monitoring cluster security policies, informer cache not synced")
} else {
dm.Logger.Print("Started to monitor cluster security policies")
}
// watch default posture
defaultPostureSynced := dm.WatchDefaultPosture()
if defaultPostureSynced == nil {
// destroy the daemon
dm.DestroyKubeArmorDaemon()
return
}
dm.Logger.Print("Started to monitor per-namespace default posture")
// watch kubearmor configmap
configMapSynced := dm.WatchConfigMap()
if configMapSynced == nil {
// destroy the daemon
dm.DestroyKubeArmorDaemon()
return
}
dm.Logger.Print("Watching for posture changes")
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
synced := cache.WaitForCacheSync(ctx.Done(), securityPoliciesSynced, defaultPostureSynced, configMapSynced)
if !synced {
dm.Logger.Err("Failed to sync Kubernetes informers")
// destroy the daemon
dm.DestroyKubeArmorDaemon()
return
}
// watch k8s pods (function never returns, must be called in a
// goroutine)
go dm.WatchK8sPods()
dm.Logger.Print("Started to monitor Pod events")
}
if dm.K8sEnabled && cfg.GlobalCfg.HostPolicy {
// watch host security policies
go dm.WatchHostSecurityPolicies(timeout)
}
if !dm.K8sEnabled && (enableContainerPolicy || cfg.GlobalCfg.HostPolicy) {
policyService := &policy.PolicyServer{
ContainerPolicyEnabled: enableContainerPolicy,
HostPolicyEnabled: cfg.GlobalCfg.HostPolicy,
}
if enableContainerPolicy {
policyService.UpdateContainerPolicy = dm.ParseAndUpdateContainerSecurityPolicy
dm.Logger.Print("Started to monitor container security policies on gRPC")
}
if cfg.GlobalCfg.HostPolicy {
policyService.UpdateHostPolicy = dm.ParseAndUpdateHostSecurityPolicy
dm.Node.PolicyEnabled = tp.KubeArmorPolicyEnabled
dm.Logger.Print("Started to monitor host security policies on gRPC")
}
pb.RegisterPolicyServiceServer(dm.Logger.LogServer, policyService)
//Enable grpc service to send kubearmor data to client in unorchestrated mode
probe := &Probe{}
probe.GetContainerData = dm.SetProbeContainerData
pb.RegisterProbeServiceServer(dm.Logger.LogServer, probe)
dm.SetHealthStatus(pb.PolicyService_ServiceDesc.ServiceName, grpc_health_v1.HealthCheckResponse_SERVING)
dm.SetHealthStatus(pb.ProbeService_ServiceDesc.ServiceName, grpc_health_v1.HealthCheckResponse_SERVING)
}
reflection.Register(dm.Logger.LogServer) // Helps grpc clients list out what all svc/endpoints available
// serve log feeds
go dm.ServeLogFeeds()
dm.Logger.Print("Started to serve gRPC-based log feeds")
dm.SetHealthStatus(pb.LogService_ServiceDesc.ServiceName, grpc_health_v1.HealthCheckResponse_SERVING)
// == //
go dm.SetKarmorData()
dm.Logger.Print("Initialized KubeArmor")
// == //
if cfg.GlobalCfg.KVMAgent || !dm.K8sEnabled {
// Restore and apply all kubearmor host security policies
dm.restoreKubeArmorPolicies()
}
// == //
// Init KvmAgent
if cfg.GlobalCfg.KVMAgent {
// initialize kvm agent
if !dm.InitKVMAgent() {
dm.Logger.Err("Failed to initialize KVM Agent")
// destroy the daemon
dm.DestroyKubeArmorDaemon()
return
}
dm.Logger.Print("Initialized KVM Agent")
// connect to KVM Service
go dm.ConnectToKVMService()
dm.Logger.Print("Started to keep the connection to KVM Service")
}
// == //
if !cfg.GlobalCfg.CoverageTest {
// listen for interrupt signals
sigChan := GetOSSigChannel()
<-sigChan
dm.Logger.Print("Got a signal to terminate KubeArmor")
}
// destroy the daemon
dm.DestroyKubeArmorDaemon()
}
func (dm *KubeArmorDaemon) checkNRIAvailability() bool {
// Check if nri socket is set, if not then auto detect
if cfg.GlobalCfg.NRISocket == "" {
if kl.GetNRISocket("") != "" {
cfg.GlobalCfg.NRISocket = kl.GetNRISocket("")
} else {
dm.Logger.Warnf("Error while looking for NRI socket file")
return false
}
} else {
// NRI socket supplied by user, check for existence
_, err := os.Stat(cfg.GlobalCfg.NRISocket)
if err != nil {
dm.Logger.Warnf("Error while looking for NRI socket file %s", err.Error())
return false
}
}
return true
}
// SPDX-License-Identifier: Apache-2.0
// Copyright 2021 Authors of KubeArmor
package core
import (
"context"
"encoding/json"
"fmt"
"io"
"os"
"reflect"
"sort"
"strconv"
"strings"
"time"
kl "github.com/kubearmor/KubeArmor/KubeArmor/common"
cfg "github.com/kubearmor/KubeArmor/KubeArmor/config"
kg "github.com/kubearmor/KubeArmor/KubeArmor/log"
"github.com/kubearmor/KubeArmor/KubeArmor/monitor"
tp "github.com/kubearmor/KubeArmor/KubeArmor/types"
ksp "github.com/kubearmor/KubeArmor/pkg/KubeArmorController/api/security.kubearmor.com/v1"
kspinformer "github.com/kubearmor/KubeArmor/pkg/KubeArmorController/client/informers/externalversions"
pb "github.com/kubearmor/KubeArmor/protobuf"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/informers"
"k8s.io/client-go/tools/cache"
)
const (
KubeArmorPolicy string = "KubeArmorPolicy"
KubeArmorClusterPolicy string = "KubeArmorClusterPolicy"
InOperator string = "In"
NotInOperator string = "NotIn"
NamespaceKey string = "namespace"
LabelKey string = "label"
// Event Types
addEvent string = "ADDED"
updateEvent string = "MODIFIED"
deleteEvent string = "DELETED"
)
// ================= //
// == Node Update == //
// ================= //
// HandleNodeAnnotations Handle Node Annotations i.e, set host visibility based on annotations, enable/disable policy
func (dm *KubeArmorDaemon) HandleNodeAnnotations(node *tp.Node) {
if _, ok := node.Annotations["kubearmor-policy"]; !ok {
node.Annotations["kubearmor-policy"] = "enabled"
}
if node.Annotations["kubearmor-policy"] != "enabled" && node.Annotations["kubearmor-policy"] != "disabled" && node.Annotations["kubearmor-policy"] != "audited" {
node.Annotations["kubearmor-policy"] = "enabled"
}
// == LSM == //
var lsm string
// Check if enforcer is set in the node annotations
if v, ok := node.Labels["kubearmor.io/enforcer"]; ok {
lsm = v
} else { // Read the lsm from the system
lsmByteData, err := os.ReadFile("/sys/kernel/security/lsm")
if err != nil && !os.IsNotExist(err) {
kg.Errf("Failed to read /sys/kernel/security/lsm (%s)", err.Error())
} else if len(lsmByteData) == 0 {
kg.Err("Failed to read /sys/kernel/security/lsm: empty file")
}
lsm = string(lsmByteData)
}
hasAppArmor := strings.Contains(lsm, "apparmor")
hasSelinux := strings.Contains(lsm, "selinux")
hasBPF := strings.Contains(lsm, "bpf")
if !hasBPF && !hasSelinux && !hasAppArmor {
// exception: neither AppArmor, SELinux or BPF
if node.Annotations["kubearmor-policy"] == "enabled" {
node.Annotations["kubearmor-policy"] = "audited"
}
}
if kl.IsInK8sCluster() && hasSelinux {
// exception: KubeArmor in a daemonset even though SELinux is enabled
if node.Annotations["kubearmor-policy"] == "enabled" {
node.Annotations["kubearmor-policy"] = "audited"
}
}
if node.Annotations["kubearmor-policy"] == "enabled" {
node.PolicyEnabled = tp.KubeArmorPolicyEnabled
} else if node.Annotations["kubearmor-policy"] == "audited" {
node.PolicyEnabled = tp.KubeArmorPolicyAudited
} else { // disabled
node.PolicyEnabled = tp.KubeArmorPolicyDisabled
}
if _, ok := node.Annotations["kubearmor-visibility"]; !ok {
node.Annotations["kubearmor-visibility"] = cfg.GlobalCfg.HostVisibility
}
for _, visibility := range strings.Split(node.Annotations["kubearmor-visibility"], ",") {
if visibility == "process" {
node.ProcessVisibilityEnabled = true
} else if visibility == "file" {
node.FileVisibilityEnabled = true
} else if visibility == "network" {
node.NetworkVisibilityEnabled = true
} else if visibility == "capabilities" {
node.CapabilitiesVisibilityEnabled = true
}
}
}
func (dm *KubeArmorDaemon) checkAndUpdateNode(item *corev1.Node) {
node := tp.Node{}
node.ClusterName = cfg.GlobalCfg.Cluster
node.NodeName = cfg.GlobalCfg.Host
for _, address := range item.Status.Addresses {
if address.Type == "InternalIP" {
node.NodeIP = address.Address
break
}
}
node.Annotations = map[string]string{}
node.Labels = map[string]string{}
node.Identities = []string{}
// update annotations
for k, v := range item.ObjectMeta.Annotations {
node.Annotations[k] = v
}
// update labels and identities
for k, v := range item.ObjectMeta.Labels {
node.Labels[k] = v
node.Identities = append(node.Identities, k+"="+v)
}
sort.Slice(node.Identities, func(i, j int) bool {
return node.Identities[i] < node.Identities[j]
})
// node info
node.Architecture = item.Status.NodeInfo.Architecture
node.OperatingSystem = item.Status.NodeInfo.OperatingSystem
node.OSImage = item.Status.NodeInfo.OSImage
node.KernelVersion = item.Status.NodeInfo.KernelVersion
node.KubeletVersion = item.Status.NodeInfo.KubeletVersion
// container runtime
node.ContainerRuntimeVersion = item.Status.NodeInfo.ContainerRuntimeVersion
dm.HandleNodeAnnotations(&node)
// update node info
dm.NodeLock.Lock()
dm.Node = node
dm.NodeLock.Unlock()
}
// WatchK8sNodes Function
func (dm *KubeArmorDaemon) WatchK8sNodes() {
kg.Printf("GlobalCfg.Host=%s, KUBEARMOR_NODENAME=%s", cfg.GlobalCfg.Host, os.Getenv("KUBEARMOR_NODENAME"))
nodeName := os.Getenv("KUBEARMOR_NODENAME")
if nodeName == "" {
nodeName = cfg.GlobalCfg.Host
}
factory := informers.NewSharedInformerFactoryWithOptions(
K8s.K8sClient,
0,
informers.WithTweakListOptions(func(options *metav1.ListOptions) {
options.FieldSelector = fmt.Sprintf("metadata.name=%s", nodeName)
}),
)
informer := factory.Core().V1().Nodes().Informer()
if _, err := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
if item, ok := obj.(*corev1.Node); ok {
dm.checkAndUpdateNode(item)
}
},
UpdateFunc: func(oldObj, newObj interface{}) {
if item, ok := newObj.(*corev1.Node); ok {
dm.checkAndUpdateNode(item)
}
},
}); err != nil {
kg.Err("Couldn't Start Watching node information")
return
}
go factory.Start(StopChan)
factory.WaitForCacheSync(StopChan)
kg.Print("Started watching node information")
}
// ================ //
// == Pod Update == //
// ================ //
// UpdateEndPointWithPod Function
func (dm *KubeArmorDaemon) UpdateEndPointWithPod(action string, pod tp.K8sPod) {
if action == addEvent {
// create a new endpoint
newPoint := tp.EndPoint{}
newPoint.NamespaceName = pod.Metadata["namespaceName"]
newPoint.EndPointName = pod.Metadata["podName"]
newPoint.Labels = map[string]string{}
newPoint.Identities = []string{"namespaceName=" + pod.Metadata["namespaceName"]}
// update labels and identities
for k, v := range pod.Labels {
newPoint.Labels[k] = v
newPoint.Identities = append(newPoint.Identities, k+"="+v)
}
sort.Slice(newPoint.Identities, func(i, j int) bool {
return newPoint.Identities[i] < newPoint.Identities[j]
})
// update policy flag
if pod.Annotations["kubearmor-policy"] == "enabled" {
newPoint.PolicyEnabled = tp.KubeArmorPolicyEnabled
} else if pod.Annotations["kubearmor-policy"] == "audited" {
newPoint.PolicyEnabled = tp.KubeArmorPolicyAudited
} else { // disabled
newPoint.PolicyEnabled = tp.KubeArmorPolicyDisabled
}
// parse annotations and update visibility flags
for _, visibility := range strings.Split(pod.Annotations["kubearmor-visibility"], ",") {
if visibility == "process" {
newPoint.ProcessVisibilityEnabled = true
} else if visibility == "file" {
newPoint.FileVisibilityEnabled = true
} else if visibility == "network" {
newPoint.NetworkVisibilityEnabled = true
} else if visibility == "capabilities" {
newPoint.CapabilitiesVisibilityEnabled = true
}
}
newPoint.Containers = []string{}
newPoint.AppArmorProfiles = []string{}
// update containers
for k := range pod.Containers {
newPoint.Containers = append(newPoint.Containers, k)
}
containersAppArmorProfiles := map[string]string{}
// update containers and apparmors
dm.ContainersLock.Lock()
for _, containerID := range newPoint.Containers {
container := dm.Containers[containerID]
container.NamespaceName = newPoint.NamespaceName
container.EndPointName = newPoint.EndPointName
if (container.Owner == tp.PodOwner{}) && (len(dm.OwnerInfo) > 0) {
if podOwnerInfo, ok := dm.OwnerInfo[container.EndPointName]; ok && (podOwnerInfo != tp.PodOwner{}) {
container.Owner = podOwnerInfo
}
}
labels := []string{}
for k, v := range newPoint.Labels {
labels = append(labels, k+"="+v)
}
container.Labels = strings.Join(labels, ",")
container.ContainerName = pod.Containers[containerID]
container.ContainerImage = pod.ContainerImages[containerID]
container.PolicyEnabled = newPoint.PolicyEnabled
container.ProcessVisibilityEnabled = newPoint.ProcessVisibilityEnabled
container.FileVisibilityEnabled = newPoint.FileVisibilityEnabled
container.NetworkVisibilityEnabled = newPoint.NetworkVisibilityEnabled
container.CapabilitiesVisibilityEnabled = newPoint.CapabilitiesVisibilityEnabled
containersAppArmorProfiles[containerID] = container.AppArmorProfile
if !kl.ContainsElement(newPoint.AppArmorProfiles, container.AppArmorProfile) {
newPoint.AppArmorProfiles = append(newPoint.AppArmorProfiles, container.AppArmorProfile)
}
// if container is privileged
if _, ok := pod.PrivilegedContainers[container.ContainerName]; ok {
container.Privileged = true
}
dm.Containers[containerID] = container
// in case if container runtime detect the container and emit that event before pod event then
// the container id will be added to NsMap with "Unknown" namespace
// therefore update the NsMap to have this container id with associated namespace
// and delete the container id from NamespacePidsMap within "Unknown" namespace
dm.HandleUnknownNamespaceNsMap(&container)
}
dm.ContainersLock.Unlock()
dm.DefaultPosturesLock.Lock()
if val, ok := dm.DefaultPostures[newPoint.NamespaceName]; ok {
newPoint.DefaultPosture = val
} else {
globalDefaultPosture := tp.DefaultPosture{
FileAction: cfg.GlobalCfg.DefaultFilePosture,
NetworkAction: cfg.GlobalCfg.DefaultNetworkPosture,
CapabilitiesAction: cfg.GlobalCfg.DefaultCapabilitiesPosture,
}
newPoint.DefaultPosture = globalDefaultPosture
}
dm.DefaultPosturesLock.Unlock()
// update security policies with the identities
newPoint.SecurityPolicies = dm.GetSecurityPolicies(newPoint)
endpoints := []tp.EndPoint{}
for k, v := range pod.Containers {
endpoint := newPoint
endpoint.Containers = []string{}
endpoint.AppArmorProfiles = []string{}
endpoint.SecurityPolicies = []tp.SecurityPolicy{}
endpoint.AppArmorProfiles = append(endpoint.AppArmorProfiles, containersAppArmorProfiles[k])
endpoint.Containers = append(endpoint.Containers, k)
endpoint.ContainerName = v
for _, secPolicy := range newPoint.SecurityPolicies {
if len(secPolicy.Spec.Selector.Containers) == 0 || kl.ContainsElement(secPolicy.Spec.Selector.Containers, v) {
endpoint.SecurityPolicies = append(endpoint.SecurityPolicies, secPolicy)
}
}
endpoints = append(endpoints, endpoint)
}
dm.EndPointsLock.Lock()
// add the endpoint into the endpoint list
dm.EndPoints = append(dm.EndPoints, endpoints...)
dm.EndPointsLock.Unlock()
if cfg.GlobalCfg.Policy {
// update security policies
for _, endpoint := range endpoints {
dm.Logger.UpdateSecurityPolicies(action, endpoint)
if newPoint.PolicyEnabled == tp.KubeArmorPolicyEnabled {
// enforce security policies
if !kl.ContainsElement(dm.SystemMonitor.UntrackedNamespaces, endpoint.NamespaceName) {
if dm.RuntimeEnforcer != nil {
dm.RuntimeEnforcer.UpdateSecurityPolicies(endpoint)
}
if dm.Presets != nil {
dm.Presets.UpdateSecurityPolicies(endpoint)
}
} else {
dm.Logger.Warnf("Policy cannot be enforced in untracked namespace %s", endpoint.NamespaceName)
}
}
}
}
} else if action == updateEvent {
newEndPoint := tp.EndPoint{}
endpoints := []tp.EndPoint{}
dm.EndPointsLock.RLock()
for _, endPoint := range dm.EndPoints {
if pod.Metadata["namespaceName"] == endPoint.NamespaceName && pod.Metadata["podName"] == endPoint.EndPointName {
endpoints = append(endpoints, endPoint)
break
}
}
dm.EndPointsLock.RUnlock()
if len(endpoints) == 0 {
// No endpoints were added as containers ID have been just added
// Same logic as ADDED
dm.UpdateEndPointWithPod(addEvent, pod)
} else {
newEndPoint.NamespaceName = pod.Metadata["namespaceName"]
newEndPoint.EndPointName = pod.Metadata["podName"]
newEndPoint.Labels = map[string]string{}
newEndPoint.Identities = []string{"namespaceName=" + pod.Metadata["namespaceName"]}
// update labels and identities
for k, v := range pod.Labels {
newEndPoint.Labels[k] = v
newEndPoint.Identities = append(newEndPoint.Identities, k+"="+v)
}
sort.Slice(newEndPoint.Identities, func(i, j int) bool {
return newEndPoint.Identities[i] < newEndPoint.Identities[j]
})
// update policy flag
if pod.Annotations["kubearmor-policy"] == "enabled" {
newEndPoint.PolicyEnabled = tp.KubeArmorPolicyEnabled
} else if pod.Annotations["kubearmor-policy"] == "audited" {
newEndPoint.PolicyEnabled = tp.KubeArmorPolicyAudited
} else { // disabled
newEndPoint.PolicyEnabled = tp.KubeArmorPolicyDisabled
}
newEndPoint.ProcessVisibilityEnabled = false
newEndPoint.FileVisibilityEnabled = false
newEndPoint.NetworkVisibilityEnabled = false
newEndPoint.CapabilitiesVisibilityEnabled = false
// parse annotations and update visibility flags
for _, visibility := range strings.Split(pod.Annotations["kubearmor-visibility"], ",") {
if visibility == "process" {
newEndPoint.ProcessVisibilityEnabled = true
} else if visibility == "file" {
newEndPoint.FileVisibilityEnabled = true
} else if visibility == "network" {
newEndPoint.NetworkVisibilityEnabled = true
} else if visibility == "capabilities" {
newEndPoint.CapabilitiesVisibilityEnabled = true
}
}
newEndPoint.Containers = []string{}
newEndPoint.AppArmorProfiles = []string{}
newEndPoint.SELinuxProfiles = []string{}
// update containers
for k := range pod.Containers {
newEndPoint.Containers = append(newEndPoint.Containers, k)
}
containersAppArmorProfiles := map[string]string{}
// update containers and apparmors
dm.ContainersLock.Lock()
for _, containerID := range newEndPoint.Containers {
container := dm.Containers[containerID]
container.NamespaceName = newEndPoint.NamespaceName
container.EndPointName = newEndPoint.EndPointName
if (container.Owner == tp.PodOwner{}) && (len(dm.OwnerInfo) > 0) {
if podOwnerInfo, ok := dm.OwnerInfo[container.EndPointName]; ok && (podOwnerInfo != tp.PodOwner{}) {
container.Owner = podOwnerInfo
}
}
labels := []string{}
for k, v := range newEndPoint.Labels {
labels = append(labels, k+"="+v)
}
container.Labels = strings.Join(labels, ",")
container.ContainerName = pod.Containers[containerID]
container.ContainerImage = pod.ContainerImages[containerID]
container.PolicyEnabled = newEndPoint.PolicyEnabled
container.ProcessVisibilityEnabled = newEndPoint.ProcessVisibilityEnabled
container.FileVisibilityEnabled = newEndPoint.FileVisibilityEnabled
container.NetworkVisibilityEnabled = newEndPoint.NetworkVisibilityEnabled
container.CapabilitiesVisibilityEnabled = newEndPoint.CapabilitiesVisibilityEnabled
containersAppArmorProfiles[containerID] = container.AppArmorProfile
if !kl.ContainsElement(newEndPoint.AppArmorProfiles, container.AppArmorProfile) {
newEndPoint.AppArmorProfiles = append(newEndPoint.AppArmorProfiles, container.AppArmorProfile)
}
// if container is privileged
if _, ok := pod.PrivilegedContainers[container.ContainerName]; ok {
container.Privileged = true
}
dm.Containers[containerID] = container
// in case if container runtime detect the container and emit that event before pod event then
// the container id will be added to NsMap with "Unknown" namespace
// therefore update the NsMap to have this container id with associated namespace
// and delete the container id from NamespacePidsMap within "Unknown" namespace
dm.HandleUnknownNamespaceNsMap(&container)
}
dm.ContainersLock.Unlock()
dm.DefaultPosturesLock.Lock()
if val, ok := dm.DefaultPostures[newEndPoint.NamespaceName]; ok {
newEndPoint.DefaultPosture = val
} else {
globalDefaultPosture := tp.DefaultPosture{
FileAction: cfg.GlobalCfg.DefaultFilePosture,
NetworkAction: cfg.GlobalCfg.DefaultNetworkPosture,
CapabilitiesAction: cfg.GlobalCfg.DefaultCapabilitiesPosture,
}
newEndPoint.DefaultPosture = globalDefaultPosture
}
dm.DefaultPosturesLock.Unlock()
// get security policies according to the updated identities
newEndPoint.SecurityPolicies = dm.GetSecurityPolicies(newEndPoint)
newendpoints := []tp.EndPoint{}
for k, v := range pod.Containers {
endpoint := newEndPoint
endpoint.Containers = []string{}
endpoint.AppArmorProfiles = []string{}
endpoint.SecurityPolicies = []tp.SecurityPolicy{}
endpoint.AppArmorProfiles = append(endpoint.AppArmorProfiles, containersAppArmorProfiles[k])
endpoint.Containers = append(endpoint.Containers, k)
endpoint.ContainerName = v
for _, secPolicy := range newEndPoint.SecurityPolicies {
if len(secPolicy.Spec.Selector.Containers) == 0 || kl.ContainsElement(secPolicy.Spec.Selector.Containers, v) {
endpoint.SecurityPolicies = append(endpoint.SecurityPolicies, secPolicy)
}
}
endpoints = append(newendpoints, endpoint)
}
dm.EndPointsLock.Lock()
idx := 0
nidx := 0
for nidx < len(endpoints) && idx < len(dm.EndPoints) {
if pod.Metadata["namespaceName"] == dm.EndPoints[idx].NamespaceName && pod.Metadata["podName"] == dm.EndPoints[idx].EndPointName && kl.ContainsElement(endpoints, dm.EndPoints[idx].ContainerName) {
dm.EndPoints[idx] = endpoints[nidx]
nidx++
}
idx++
}
dm.EndPointsLock.Unlock()
for _, endpoint := range endpoints {
if cfg.GlobalCfg.Policy {
// update security policies
dm.Logger.UpdateSecurityPolicies(action, endpoint)
if endpoint.PolicyEnabled == tp.KubeArmorPolicyEnabled {
// enforce security policies
if !kl.ContainsElement(dm.SystemMonitor.UntrackedNamespaces, endpoint.NamespaceName) {
if dm.RuntimeEnforcer != nil {
dm.RuntimeEnforcer.UpdateSecurityPolicies(endpoint)
}
if dm.Presets != nil {
dm.Presets.UpdateSecurityPolicies(endpoint)
}
} else {
dm.Logger.Warnf("Policy cannot be enforced in untracked namespace %s", endpoint.NamespaceName)
}
}
}
}
}
} else { // DELETED
dm.EndPointsLock.Lock()
idx := 0
endpointsLength := len(dm.EndPoints)
for idx < endpointsLength {
endpoint := dm.EndPoints[idx]
if pod.Metadata["namespaceName"] == endpoint.NamespaceName && pod.Metadata["podName"] == endpoint.EndPointName {
dm.EndPoints = append(dm.EndPoints[:idx], dm.EndPoints[idx+1:]...)
endpointsLength--
idx--
}
idx++
}
dm.EndPointsLock.Unlock()
}
}
// HandleUnknownNamespaceNsMap Function
func (dm *KubeArmorDaemon) HandleUnknownNamespaceNsMap(container *tp.Container) {
dm.SystemMonitor.AddContainerIDToNsMap(container.ContainerID, container.NamespaceName, container.PidNS, container.MntNS)
dm.SystemMonitor.NsMapLock.Lock()
if val, ok := dm.SystemMonitor.NamespacePidsMap["Unknown"]; ok {
for i := range val.NsKeys {
if val.NsKeys[i].MntNS == container.MntNS && val.NsKeys[i].PidNS == container.PidNS {
val.NsKeys = append(val.NsKeys[:i], val.NsKeys[i+1:]...)
break
}
}
dm.SystemMonitor.NamespacePidsMap["Unknown"] = val
}
dm.SystemMonitor.NsMapLock.Unlock()
}
func (dm *KubeArmorDaemon) handlePodEvent(event string, obj *corev1.Pod) {
if event != addEvent && event != updateEvent && event != deleteEvent {
return
}
// create a pod
pod := tp.K8sPod{}
containers := []string{}
// need this for apparmor profile
var podOwnerName string
pod.Metadata = map[string]string{}
pod.Metadata["namespaceName"] = obj.ObjectMeta.Namespace
pod.Metadata["podName"] = obj.ObjectMeta.Name
var controllerName, controller, namespace string
var err error
if event == addEvent {
controllerName, controller, namespace, err = getTopLevelOwner(obj.ObjectMeta, obj.Namespace, obj.Kind)
if err != nil {
dm.Logger.Warnf("Failed to get ownerRef (%s, %s)", obj.ObjectMeta.Name, err.Error())
}
owner := tp.PodOwner{
Name: controllerName,
Ref: controller,
Namespace: namespace,
}
dm.OwnerInfo[pod.Metadata["podName"]] = owner
podOwnerName = controllerName
}
// for event = UpdateEvent we first check pod's existence to update current dm.OwnerInfo of the pod, because when pod is in terminating state then we cannot get the owner info from it.
// we do not update owner info in terminating state. After pod is deleted we delete the owner info from the map.
_, err = K8s.K8sClient.CoreV1().Pods(namespace).Get(context.Background(), obj.ObjectMeta.Name, metav1.GetOptions{})
if err == nil && event == updateEvent {
controllerName, controller, namespace, err = getTopLevelOwner(obj.ObjectMeta, obj.Namespace, obj.Kind)
if err != nil {
dm.Logger.Warnf("Failed to get ownerRef (%s, %s)", obj.ObjectMeta.Name, err.Error())
}
owner := tp.PodOwner{
Name: controllerName,
Ref: controller,
Namespace: namespace,
}
dm.OwnerInfo[pod.Metadata["podName"]] = owner
podOwnerName = controllerName
}
//get the owner , then check if that owner has owner if...do it recusivelt until you get the no owner
pod.Annotations = map[string]string{}
for k, v := range obj.Annotations {
pod.Annotations[k] = v
}
pod.Labels = map[string]string{}
for k, v := range obj.Labels {
if k == "pod-template-hash" {
continue
}
if k == "pod-template-generation" {
continue
}
if k == "controller-revision-hash" {
continue
}
pod.Labels[k] = v
}
pod.Containers = map[string]string{}
pod.ContainerImages = map[string]string{}
for _, container := range obj.Status.ContainerStatuses {
if len(container.ContainerID) > 0 {
cid := strings.Split(container.ContainerID, "://")
if len(cid) == 2 { // always true because k8s spec defines format as '<type>://<container_id>'
containerID := cid[1]
pod.Containers[containerID] = container.Name
pod.ContainerImages[containerID] = container.Image + kl.GetSHA256ofImage(container.ImageID)
}
}
}
// == Policy == //
if _, ok := pod.Annotations["kubearmor-policy"]; !ok {
pod.Annotations["kubearmor-policy"] = "enabled"
}
if pod.Annotations["kubearmor-policy"] != "enabled" && pod.Annotations["kubearmor-policy"] != "disabled" && pod.Annotations["kubearmor-policy"] != "audited" {
pod.Annotations["kubearmor-policy"] = "enabled"
}
// == LSM == //
if dm.RuntimeEnforcer == nil {
// exception: no LSM
if pod.Annotations["kubearmor-policy"] == "enabled" {
pod.Annotations["kubearmor-policy"] = "audited"
}
} else if dm.RuntimeEnforcer != nil && dm.RuntimeEnforcer.EnforcerType == "SELinux" {
// exception: no SELinux support for containers
if pod.Annotations["kubearmor-policy"] == "enabled" {
pod.Annotations["kubearmor-policy"] = "audited"
}
}
// == Exception == //
// exception: kubernetes app
if pod.Metadata["namespaceName"] == "kube-system" {
pod.Annotations["kubearmor-policy"] = "audited"
}
// exception: cilium-operator
if _, ok := pod.Labels["io.cilium/app"]; ok {
pod.Annotations["kubearmor-policy"] = "audited"
}
// exception: kubearmor
// if _, ok := pod.Labels["kubearmor-app"]; ok {
// pod.Annotations["kubearmor-policy"] = "audited"
// }
// == Visibility == //
if _, ok := pod.Annotations["kubearmor-visibility"]; !ok {
pod.Annotations["kubearmor-visibility"] = cfg.GlobalCfg.Visibility
}
// == AppArmor == //
if event == addEvent || event == updateEvent {
exist := false
dm.K8sPodsLock.RLock()
for _, k8spod := range dm.K8sPods {
if k8spod.Metadata["namespaceName"] == pod.Metadata["namespaceName"] && k8spod.Metadata["podName"] == pod.Metadata["podName"] {
if k8spod.Annotations["kubearmor-policy"] == "patched" {
exist = true
break
}
}
}
dm.K8sPodsLock.RUnlock()
if exist {
return
}
}
pod.PrivilegedContainers = make(map[string]struct{})
pod.PrivilegedAppArmorProfiles = make(map[string]struct{})
if dm.RuntimeEnforcer != nil && dm.RuntimeEnforcer.EnforcerType == "AppArmor" {
appArmorAnnotations := map[string]string{}
updateAppArmor := false
if dm.OwnerInfo[pod.Metadata["podName"]].Name != "" {
if dm.OwnerInfo[pod.Metadata["podName"]].Ref == "StatefulSet" {
statefulset, err := K8s.K8sClient.AppsV1().StatefulSets(pod.Metadata["namespaceName"]).Get(context.Background(), podOwnerName, metav1.GetOptions{})
if err == nil {
for _, c := range statefulset.Spec.Template.Spec.Containers {
containers = append(containers, c.Name)
}
}
} else if dm.OwnerInfo[pod.Metadata["podName"]].Ref == "ReplicaSet" {
replica, err := K8s.K8sClient.AppsV1().ReplicaSets(pod.Metadata["namespaceName"]).Get(context.Background(), podOwnerName, metav1.GetOptions{})
if err == nil {
for _, c := range replica.Spec.Template.Spec.Containers {
containers = append(containers, c.Name)
}
}
} else if dm.OwnerInfo[pod.Metadata["podName"]].Ref == "DaemonSet" {
daemon, err := K8s.K8sClient.AppsV1().DaemonSets(pod.Metadata["namespaceName"]).Get(context.Background(), podOwnerName, metav1.GetOptions{})
if err == nil {
for _, c := range daemon.Spec.Template.Spec.Containers {
containers = append(containers, c.Name)
}
}
} else if dm.OwnerInfo[pod.Metadata["podName"]].Ref == "Deployment" {
deploy, err := K8s.K8sClient.AppsV1().Deployments(pod.Metadata["namespaceName"]).Get(context.Background(), podOwnerName, metav1.GetOptions{})
if err == nil {
for _, c := range deploy.Spec.Template.Spec.Containers {
containers = append(containers, c.Name)
}
}
} else if dm.OwnerInfo[pod.Metadata["podName"]].Ref == "Pod" {
pod, err := K8s.K8sClient.CoreV1().Pods(pod.Metadata["namespaceName"]).Get(context.Background(), podOwnerName, metav1.GetOptions{})
if err == nil {
for _, c := range pod.Spec.Containers {
containers = append(containers, c.Name)
}
}
} else if dm.OwnerInfo[pod.Metadata["podName"]].Ref == "Job" {
job, err := K8s.K8sClient.BatchV1().Jobs(pod.Metadata["namespaceName"]).Get(context.Background(), podOwnerName, metav1.GetOptions{})
if err == nil {
for _, c := range job.Spec.Template.Spec.Containers {
containers = append(containers, c.Name)
}
}
} else if dm.OwnerInfo[pod.Metadata["podName"]].Ref == "CronJob" {
cronJob, err := K8s.K8sClient.BatchV1().CronJobs(pod.Metadata["namespaceName"]).Get(context.Background(), podOwnerName, metav1.GetOptions{})
if err == nil {
for _, c := range cronJob.Spec.JobTemplate.Spec.Template.Spec.Containers {
containers = append(containers, c.Name)
}
}
}
}
for k, v := range pod.Annotations {
if strings.HasPrefix(k, "container.apparmor.security.beta.kubernetes.io") {
if v == "unconfined" {
containerName := strings.Split(k, "/")[1]
appArmorAnnotations[containerName] = v
} else {
containerName := strings.Split(k, "/")[1]
appArmorAnnotations[containerName] = strings.Split(v, "/")[1]
}
}
}
for _, container := range obj.Spec.Containers {
var privileged bool
// store privileged containers
if container.SecurityContext != nil &&
((container.SecurityContext.Privileged != nil && *container.SecurityContext.Privileged) ||
(container.SecurityContext.Capabilities != nil && len(container.SecurityContext.Capabilities.Add) > 0)) {
pod.PrivilegedContainers[container.Name] = struct{}{}
privileged = true
}
profileName := "kubearmor-" + pod.Metadata["namespaceName"] + "-" + podOwnerName + "-" + container.Name
if _, ok := appArmorAnnotations[container.Name]; !ok && kl.ContainsElement(containers, container.Name) {
appArmorAnnotations[container.Name] = profileName
updateAppArmor = true
// if the container is privileged or it has more than one capabilities added
// handle the apparmor profile generation with privileged rules
}
if privileged {
// container name is unique for all containers in a pod
pod.PrivilegedAppArmorProfiles[profileName] = struct{}{}
}
}
if event == addEvent {
// update apparmor profiles
dm.RuntimeEnforcer.UpdateAppArmorProfiles(pod.Metadata["podName"], addEvent, appArmorAnnotations, pod.PrivilegedAppArmorProfiles)
if updateAppArmor && pod.Annotations["kubearmor-policy"] == "enabled" && dm.OwnerInfo[pod.Metadata["podName"]].Ref != "Pod" {
// patch deployments only when kubearmor-controller is not present
if dm.OwnerInfo[pod.Metadata["podName"]].Name != "" && cfg.GlobalCfg.AnnotateResources {
deploymentName := dm.OwnerInfo[pod.Metadata["podName"]].Name
// patch the deployment with apparmor annotations
if err := K8s.PatchResourceWithAppArmorAnnotations(pod.Metadata["namespaceName"], deploymentName, appArmorAnnotations, dm.OwnerInfo[pod.Metadata["podName"]].Ref); err != nil {
dm.Logger.Errf("Failed to update AppArmor Annotations (%s/%s/%s, %s)", pod.Metadata["namespaceName"], deploymentName, pod.Metadata["podName"], err.Error())
} else {
dm.Logger.Printf("Patched AppArmor Annotations (%s/%s/%s)", pod.Metadata["namespaceName"], deploymentName, pod.Metadata["podName"])
}
pod.Annotations["kubearmor-policy"] = "patched"
}
}
} else if event == updateEvent {
for _, k8spod := range dm.K8sPods {
if k8spod.Metadata["namespaceName"] == pod.Metadata["namespaceName"] && k8spod.Metadata["podName"] == pod.Metadata["podName"] {
prevPolicyEnabled := "disabled"
if val, ok := k8spod.Annotations["kubearmor-policy"]; ok {
prevPolicyEnabled = val
}
if updateAppArmor && prevPolicyEnabled != "enabled" && pod.Annotations["kubearmor-policy"] == "enabled" && dm.OwnerInfo[pod.Metadata["podName"]].Ref != "Pod" {
// patch deployments only when kubearmor-controller is not present
if dm.OwnerInfo[pod.Metadata["podName"]].Name != "" && cfg.GlobalCfg.AnnotateResources {
deploymentName := dm.OwnerInfo[pod.Metadata["podName"]].Name
// patch the deployment with apparmor annotations
if err := K8s.PatchResourceWithAppArmorAnnotations(pod.Metadata["namespaceName"], deploymentName, appArmorAnnotations, dm.OwnerInfo[pod.Metadata["podName"]].Ref); err != nil {
dm.Logger.Errf("Failed to update AppArmor Annotations (%s/%s/%s, %s)", pod.Metadata["namespaceName"], deploymentName, pod.Metadata["podName"], err.Error())
} else {
dm.Logger.Printf("Patched AppArmor Annotations (%s/%s/%s)", pod.Metadata["namespaceName"], deploymentName, pod.Metadata["podName"])
}
pod.Annotations["kubearmor-policy"] = "patched"
}
}
break
}
}
} else if event == deleteEvent {
// update apparmor profiles
dm.RuntimeEnforcer.UpdateAppArmorProfiles(pod.Metadata["podName"], deleteEvent, appArmorAnnotations, pod.PrivilegedAppArmorProfiles)
}
}
dm.K8sPodsLock.Lock()
if event == addEvent {
newPod := true
for _, k8spod := range dm.K8sPods {
if k8spod.Metadata["namespaceName"] == pod.Metadata["namespaceName"] && k8spod.Metadata["podName"] == pod.Metadata["podName"] {
newPod = false
break
}
}
if newPod {
dm.K8sPods = append(dm.K8sPods, pod)
} else {
// Kubernetes can send us 'ADDED' events for a pod we
// already know about when our Kubernetes watch request
// restarts, so treat that like a 'MODIFIED' event
// instead
event = updateEvent
}
}
if event == updateEvent {
for idx, k8spod := range dm.K8sPods {
if k8spod.Metadata["namespaceName"] == pod.Metadata["namespaceName"] && k8spod.Metadata["podName"] == pod.Metadata["podName"] {
dm.K8sPods[idx] = pod
break
}
}
} else if event == deleteEvent {
for idx, k8spod := range dm.K8sPods {
if k8spod.Metadata["namespaceName"] == pod.Metadata["namespaceName"] && k8spod.Metadata["podName"] == pod.Metadata["podName"] {
dm.K8sPods = append(dm.K8sPods[:idx], dm.K8sPods[idx+1:]...)
delete(dm.OwnerInfo, pod.Metadata["podName"])
break
}
}
}
dm.K8sPodsLock.Unlock()
if pod.Annotations["kubearmor-policy"] == "patched" {
dm.Logger.Printf("Detected a Pod (patched/%s/%s)", pod.Metadata["namespaceName"], pod.Metadata["podName"])
return
} else {
dm.Logger.Printf("Detected a Pod (%s/%s/%s)", strings.ToLower(event), pod.Metadata["namespaceName"], pod.Metadata["podName"])
}
// update a endpoint corresponding to the pod
dm.UpdateEndPointWithPod(event, pod)
}
// WatchK8sPods Function
func (dm *KubeArmorDaemon) WatchK8sPods() {
if !kl.IsK8sEnv() {
dm.Logger.Print("not in a k8s environment")
return
}
nodeName := os.Getenv("KUBEARMOR_NODENAME")
if nodeName == "" {
nodeName = cfg.GlobalCfg.Host
}
nodeFieldSelector := informers.WithTweakListOptions(func(opts *metav1.ListOptions) {
opts.FieldSelector = fmt.Sprintf("spec.nodeName=%s", nodeName)
})
factory := informers.NewSharedInformerFactoryWithOptions(K8s.K8sClient, 0, nodeFieldSelector)
informer := factory.Core().V1().Pods().Informer()
var err error
if _, err = informer.AddEventHandler(
cache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
if pod, ok := obj.(*corev1.Pod); ok {
dm.handlePodEvent(addEvent, pod)
}
},
UpdateFunc: func(_, newObj interface{}) {
if pod, ok := newObj.(*corev1.Pod); ok {
dm.handlePodEvent(updateEvent, pod)
}
},
DeleteFunc: func(obj interface{}) {
if pod, ok := obj.(*corev1.Pod); ok {
dm.handlePodEvent(deleteEvent, pod)
}
},
},
); err != nil {
dm.Logger.Warnf("Error starting pod informer=%s", err)
return
}
go factory.Start(StopChan)
dm.Logger.Print("Started watching pod information")
}
// updateNamespaceListforCSP - in case of NotIn operator for namespace key, a new ns might be added later
// and here we will update namespaceList for CSP
func updateNamespaceListforCSP(policy *tp.SecurityPolicy) {
if len(policy.Spec.Selector.Identities) > 0 {
// if is not a Cluster policy, return
return
}
hasInOperator := false
excludedNamespaces := make(map[string]bool)
for _, matchExpression := range policy.Spec.Selector.MatchExpressions {
if matchExpression.Key == "namespace" {
if matchExpression.Operator == "In" {
hasInOperator = true
for _, value := range matchExpression.Values {
if !kl.ContainsElement(policy.Spec.Selector.NamespaceList, value) {
policy.Spec.Selector.NamespaceList = append(policy.Spec.Selector.NamespaceList, value)
}
}
} else if matchExpression.Operator == "NotIn" && !hasInOperator {
for _, value := range matchExpression.Values {
excludedNamespaces[value] = true
}
}
}
}
// this logic will also work when selector is not defined, and policy rule will be applied across all the namespaces
if !hasInOperator {
nsList, err := K8s.K8sClient.CoreV1().Namespaces().List(context.Background(), metav1.ListOptions{})
if err != nil {
kg.Err("unable to fetch namespace list")
return
}
for _, ns := range nsList.Items {
if _, ok := excludedNamespaces[ns.Name]; !ok && !kl.ContainsElement(policy.Spec.Selector.NamespaceList, ns.Name) {
policy.Spec.Selector.NamespaceList = append(policy.Spec.Selector.NamespaceList, ns.Name)
}
}
}
}
// ============================ //
// == Security Policy Update == //
// ============================ //
// GetSecurityPolicies Function
func (dm *KubeArmorDaemon) GetSecurityPolicies(endPoint tp.EndPoint) []tp.SecurityPolicy {
dm.SecurityPoliciesLock.RLock()
defer dm.SecurityPoliciesLock.RUnlock()
secPolicies := []tp.SecurityPolicy{}
for _, policy := range dm.SecurityPolicies {
updateNamespaceListforCSP(&policy)
// match ksp || csp
if (kl.MatchIdentities(policy.Spec.Selector.Identities, endPoint.Identities) && kl.MatchExpIdentities(policy.Spec.Selector, endPoint.Identities)) ||
(kl.ContainsElement(policy.Spec.Selector.NamespaceList, endPoint.NamespaceName) && kl.MatchExpIdentities(policy.Spec.Selector, endPoint.Identities)) {
secPolicy := tp.SecurityPolicy{}
if err := kl.Clone(policy, &secPolicy); err != nil {
dm.Logger.Errf("Failed to clone a policy (%s)", err.Error())
}
secPolicies = append(secPolicies, secPolicy)
}
}
return secPolicies
}
func containsPolicy(endPointPolicies []tp.SecurityPolicy, secPolicy tp.SecurityPolicy) bool {
for _, policy := range endPointPolicies {
if policy.Metadata["policyName"] == secPolicy.Metadata["policyName"] {
return true
}
}
return false
}
// UpdateSecurityPolicy Function
func (dm *KubeArmorDaemon) UpdateSecurityPolicy(action string, secPolicyType string, secPolicy tp.SecurityPolicy) {
dm.EndPointsLock.RLock()
endPointsLength := len(dm.EndPoints)
dm.EndPointsLock.RUnlock()
for idx := 0; idx < endPointsLength; idx++ {
dm.EndPointsLock.RLock()
endPoint := dm.EndPoints[idx]
dm.EndPointsLock.RUnlock()
// update a security policy
if secPolicyType == KubeArmorPolicy {
if len(secPolicy.Spec.Selector.Containers) == 0 || kl.ContainsElement(secPolicy.Spec.Selector.Containers, endPoint.ContainerName) {
if action == addEvent {
if kl.MatchIdentities(secPolicy.Spec.Selector.Identities, endPoint.Identities) && kl.MatchExpIdentities(secPolicy.Spec.Selector, endPoint.Identities) {
// add a new security policy if it doesn't exist
new := true
for _, policy := range endPoint.SecurityPolicies {
if policy.Metadata["namespaceName"] == secPolicy.Metadata["namespaceName"] && policy.Metadata["policyName"] == secPolicy.Metadata["policyName"] {
new = false
break
}
}
if new {
endPoint.SecurityPolicies = append(endPoint.SecurityPolicies, secPolicy)
}
}
} else if action == updateEvent {
// in case new labels are added in the policy, check if identities match, if yes, add policy in endPoint
addNewPolicy := true
for idxP, policy := range endPoint.SecurityPolicies {
if policy.Metadata["namespaceName"] == secPolicy.Metadata["namespaceName"] && policy.Metadata["policyName"] == secPolicy.Metadata["policyName"] {
if !(kl.MatchIdentities(secPolicy.Spec.Selector.Identities, endPoint.Identities) && kl.MatchExpIdentities(secPolicy.Spec.Selector, endPoint.Identities)) {
endPoint.SecurityPolicies = append(endPoint.SecurityPolicies[:idxP], endPoint.SecurityPolicies[idxP+1:]...)
addNewPolicy = false
break
}
endPoint.SecurityPolicies[idxP] = secPolicy
addNewPolicy = false
break
}
}
// check identities before adding poilicies
if addNewPolicy && kl.MatchIdentities(secPolicy.Spec.Selector.Identities, endPoint.Identities) && kl.MatchExpIdentities(secPolicy.Spec.Selector, endPoint.Identities) {
endPoint.SecurityPolicies = append(endPoint.SecurityPolicies, secPolicy)
}
} else if action == deleteEvent {
// remove the given policy from the security policy list of this endpoint
for idxP, policy := range endPoint.SecurityPolicies {
if policy.Metadata["namespaceName"] == secPolicy.Metadata["namespaceName"] && policy.Metadata["policyName"] == secPolicy.Metadata["policyName"] {
endPoint.SecurityPolicies = append(endPoint.SecurityPolicies[:idxP], endPoint.SecurityPolicies[idxP+1:]...)
break
}
}
}
dm.EndPointsLock.Lock()
dm.EndPoints[idx] = endPoint
dm.EndPointsLock.Unlock()
if cfg.GlobalCfg.Policy {
// update security policies
dm.Logger.UpdateSecurityPolicies("UPDATED", endPoint)
if dm.EndPoints[idx].PolicyEnabled == tp.KubeArmorPolicyEnabled {
// enforce security policies
if !kl.ContainsElement(dm.SystemMonitor.UntrackedNamespaces, dm.EndPoints[idx].NamespaceName) {
if dm.RuntimeEnforcer != nil {
dm.RuntimeEnforcer.UpdateSecurityPolicies(dm.EndPoints[idx])
}
if dm.Presets != nil {
dm.Presets.UpdateSecurityPolicies(dm.EndPoints[idx])
}
} else {
dm.Logger.Warnf("Policy cannot be enforced in untracked namespace %s", dm.EndPoints[idx].NamespaceName)
}
}
}
}
} else if secPolicyType == KubeArmorClusterPolicy {
// additional OR check added with containsPolicy() is when this endPoint's ns is removed from secPolicy.Spec.Selector.MatchExpressions[i].Values
// due to which secPolicy.Spec.Selector.NamespaceList will not have the removed ns
if kl.ContainsElement(secPolicy.Spec.Selector.NamespaceList, endPoint.NamespaceName) || containsPolicy(endPoint.SecurityPolicies, secPolicy) {
if action == addEvent {
if kl.ContainsElement(secPolicy.Spec.Selector.NamespaceList, endPoint.NamespaceName) && kl.MatchExpIdentities(secPolicy.Spec.Selector, endPoint.Identities) {
// add a new security policy if it doesn't exist
new := true
for _, policy := range endPoint.SecurityPolicies {
if policy.Metadata["policyName"] == secPolicy.Metadata["policyName"] {
new = false
break
}
}
if new {
endPoint.SecurityPolicies = append(endPoint.SecurityPolicies, secPolicy)
}
}
} else if action == updateEvent {
// when policy is modified and new ns is added in secPolicy.Spec.Selector.MatchExpressions[i].Values
addNewPolicy := true
for idxP, policy := range endPoint.SecurityPolicies {
if policy.Metadata["policyName"] == secPolicy.Metadata["policyName"] {
if !(kl.ContainsElement(secPolicy.Spec.Selector.NamespaceList, endPoint.NamespaceName) && kl.MatchExpIdentities(secPolicy.Spec.Selector, endPoint.Identities)) {
// when policy is modified and this endPoint's ns is removed from secPolicy.Spec.Selector.MatchExpressions[i].Values
endPoint.SecurityPolicies = append(endPoint.SecurityPolicies[:idxP], endPoint.SecurityPolicies[idxP+1:]...)
addNewPolicy = false
break
}
endPoint.SecurityPolicies[idxP] = secPolicy
addNewPolicy = false
break
}
}
// always check identities before adding poilicies
if addNewPolicy && kl.ContainsElement(secPolicy.Spec.Selector.NamespaceList, endPoint.NamespaceName) && kl.MatchExpIdentities(secPolicy.Spec.Selector, endPoint.Identities) {
endPoint.SecurityPolicies = append(endPoint.SecurityPolicies, secPolicy)
}
} else if action == deleteEvent {
// remove the given policy from the security policy list of this endpoint
for idxP, policy := range endPoint.SecurityPolicies {
if policy.Metadata["policyName"] == secPolicy.Metadata["policyName"] {
endPoint.SecurityPolicies = append(endPoint.SecurityPolicies[:idxP], endPoint.SecurityPolicies[idxP+1:]...)
break
}
}
}
dm.EndPointsLock.Lock()
dm.EndPoints[idx] = endPoint
dm.EndPointsLock.Unlock()
if cfg.GlobalCfg.Policy {
// update security policies
dm.Logger.UpdateSecurityPolicies("UPDATED", endPoint)
if dm.EndPoints[idx].PolicyEnabled == tp.KubeArmorPolicyEnabled {
// enforce security policies
if !kl.ContainsElement(dm.SystemMonitor.UntrackedNamespaces, dm.EndPoints[idx].NamespaceName) {
if dm.RuntimeEnforcer != nil {
dm.RuntimeEnforcer.UpdateSecurityPolicies(dm.EndPoints[idx])
}
if dm.Presets != nil {
dm.Presets.UpdateSecurityPolicies(dm.EndPoints[idx])
}
} else {
dm.Logger.Warnf("Policy cannot be enforced in untracked namespace %s", dm.EndPoints[idx].NamespaceName)
}
}
}
}
}
}
}
// CreateSecurityPolicy - creates `KubeArmorPolicy` & `KubeArmorClusterPolicy` object from crd
func (dm *KubeArmorDaemon) CreateSecurityPolicy(policyType string, securityPolicy interface{}) (secPolicy tp.SecurityPolicy, err error) {
var namespace, name string
if policyType == KubeArmorPolicy {
kubearmorPolicy := securityPolicy.(ksp.KubeArmorPolicy)
namespace = kubearmorPolicy.Namespace
name = kubearmorPolicy.Name
if err := kl.Clone(kubearmorPolicy.Spec, &secPolicy.Spec); err != nil {
dm.Logger.Errf("Failed to clone a spec (%s)", err.Error())
return tp.SecurityPolicy{}, err
}
// add identities
secPolicy.Spec.Selector.Identities = []string{"namespaceName=" + namespace}
for k, v := range secPolicy.Spec.Selector.MatchLabels {
if k == "kubearmor.io/container.name" {
if len(v) > 2 {
containerArray := v[1 : len(v)-1]
containers := strings.Split(containerArray, ",")
for _, container := range containers {
if len(container) > 0 {
secPolicy.Spec.Selector.Containers = append(secPolicy.Spec.Selector.Containers, strings.TrimSpace(container))
}
}
}
} else {
secPolicy.Spec.Selector.Identities = append(secPolicy.Spec.Selector.Identities, k+"="+v)
}
}
hasInOperator := false
for _, matchExpression := range secPolicy.Spec.Selector.MatchExpressions {
if matchExpression.Key == LabelKey {
if matchExpression.Operator == InOperator {
for _, label := range matchExpression.Values {
hasInOperator = true
secPolicy.Spec.Selector.MatchExpIdentities = append(secPolicy.Spec.Selector.MatchExpIdentities, label)
}
} else if matchExpression.Operator == NotInOperator && !hasInOperator {
for _, label := range matchExpression.Values {
secPolicy.Spec.Selector.NonIdentities = append(secPolicy.Spec.Selector.NonIdentities, label)
}
}
}
}
sort.Slice(secPolicy.Spec.Selector.Identities, func(i, j int) bool {
return secPolicy.Spec.Selector.Identities[i] < secPolicy.Spec.Selector.Identities[j]
})
sort.Slice(secPolicy.Spec.Selector.MatchExpIdentities, func(i, j int) bool {
return secPolicy.Spec.Selector.MatchExpIdentities[i] < secPolicy.Spec.Selector.MatchExpIdentities[j]
})
sort.Slice(secPolicy.Spec.Selector.NonIdentities, func(i, j int) bool {
return secPolicy.Spec.Selector.NonIdentities[i] < secPolicy.Spec.Selector.NonIdentities[j]
})
} else if policyType == KubeArmorClusterPolicy {
kubearmorClusterPolicy := securityPolicy.(ksp.KubeArmorClusterPolicy)
namespace = kubearmorClusterPolicy.Namespace
name = kubearmorClusterPolicy.Name
if err := kl.Clone(kubearmorClusterPolicy.Spec, &secPolicy.Spec); err != nil {
dm.Logger.Errf("Failed to clone a spec (%s)", err.Error())
return tp.SecurityPolicy{}, err
}
hasNsInOperator := false
hasLabelInOperator := false
excludedNamespaces := make(map[string]bool)
for _, matchExpression := range secPolicy.Spec.Selector.MatchExpressions {
if matchExpression.Key == NamespaceKey {
if matchExpression.Operator == InOperator {
hasNsInOperator = true
secPolicy.Spec.Selector.NamespaceList = append(secPolicy.Spec.Selector.NamespaceList, matchExpression.Values...)
} else if matchExpression.Operator == NotInOperator && !hasNsInOperator {
for _, value := range matchExpression.Values {
excludedNamespaces[value] = true
}
}
} else if matchExpression.Key == LabelKey {
if matchExpression.Operator == InOperator {
for _, label := range matchExpression.Values {
hasLabelInOperator = true
secPolicy.Spec.Selector.MatchExpIdentities = append(secPolicy.Spec.Selector.MatchExpIdentities, label)
}
} else if matchExpression.Operator == NotInOperator && !hasLabelInOperator {
for _, label := range matchExpression.Values {
secPolicy.Spec.Selector.NonIdentities = append(secPolicy.Spec.Selector.NonIdentities, label)
}
}
}
}
sort.Slice(secPolicy.Spec.Selector.MatchExpIdentities, func(i, j int) bool {
return secPolicy.Spec.Selector.MatchExpIdentities[i] < secPolicy.Spec.Selector.MatchExpIdentities[j]
})
sort.Slice(secPolicy.Spec.Selector.NonIdentities, func(i, j int) bool {
return secPolicy.Spec.Selector.NonIdentities[i] < secPolicy.Spec.Selector.NonIdentities[j]
})
// this logic will also work when selector is not defined, and policy rule will be applied across all the namespaces
if !hasNsInOperator {
nsList, err := K8s.K8sClient.CoreV1().Namespaces().List(context.Background(), metav1.ListOptions{})
if err != nil {
kg.Err("unable to fetch namespace list")
return tp.SecurityPolicy{}, err
}
for _, ns := range nsList.Items {
if _, ok := excludedNamespaces[ns.Name]; !ok {
secPolicy.Spec.Selector.NamespaceList = append(secPolicy.Spec.Selector.NamespaceList, ns.Name)
}
}
}
}
secPolicy.Metadata = map[string]string{}
secPolicy.Metadata["namespaceName"] = namespace
secPolicy.Metadata["policyName"] = name
kl.ObjCommaExpandFirstDupOthers(&secPolicy.Spec.Network.MatchProtocols)
kl.ObjCommaExpandFirstDupOthers(&secPolicy.Spec.Capabilities.MatchCapabilities)
switch secPolicy.Spec.Action {
case "allow":
secPolicy.Spec.Action = "Allow"
case "audit":
secPolicy.Spec.Action = "Audit"
case "block":
secPolicy.Spec.Action = "Block"
case "":
secPolicy.Spec.Action = "Block" // by default
}
// add severities, tags, messages, and actions
if len(secPolicy.Spec.Process.MatchPaths) > 0 {
for idx, path := range secPolicy.Spec.Process.MatchPaths {
if path.Severity == 0 {
if secPolicy.Spec.Process.Severity != 0 {
secPolicy.Spec.Process.MatchPaths[idx].Severity = secPolicy.Spec.Process.Severity
} else {
secPolicy.Spec.Process.MatchPaths[idx].Severity = secPolicy.Spec.Severity
}
}
if len(path.Tags) == 0 {
if len(secPolicy.Spec.Process.Tags) > 0 {
secPolicy.Spec.Process.MatchPaths[idx].Tags = secPolicy.Spec.Process.Tags
} else {
secPolicy.Spec.Process.MatchPaths[idx].Tags = secPolicy.Spec.Tags
}
}
if len(path.Message) == 0 {
if len(secPolicy.Spec.Process.Message) > 0 {
secPolicy.Spec.Process.MatchPaths[idx].Message = secPolicy.Spec.Process.Message
} else {
secPolicy.Spec.Process.MatchPaths[idx].Message = secPolicy.Spec.Message
}
}
if len(path.Action) == 0 {
if len(secPolicy.Spec.Process.Action) > 0 {
secPolicy.Spec.Process.MatchPaths[idx].Action = secPolicy.Spec.Process.Action
} else {
secPolicy.Spec.Process.MatchPaths[idx].Action = secPolicy.Spec.Action
}
}
}
}
if len(secPolicy.Spec.Process.MatchDirectories) > 0 {
for idx, dir := range secPolicy.Spec.Process.MatchDirectories {
if dir.Severity == 0 {
if secPolicy.Spec.Process.Severity != 0 {
secPolicy.Spec.Process.MatchDirectories[idx].Severity = secPolicy.Spec.Process.Severity
} else {
secPolicy.Spec.Process.MatchDirectories[idx].Severity = secPolicy.Spec.Severity
}
}
if len(dir.Tags) == 0 {
if len(secPolicy.Spec.Process.Tags) > 0 {
secPolicy.Spec.Process.MatchDirectories[idx].Tags = secPolicy.Spec.Process.Tags
} else {
secPolicy.Spec.Process.MatchDirectories[idx].Tags = secPolicy.Spec.Tags
}
}
if len(dir.Message) == 0 {
if len(secPolicy.Spec.Process.Message) > 0 {
secPolicy.Spec.Process.MatchDirectories[idx].Message = secPolicy.Spec.Process.Message
} else {
secPolicy.Spec.Process.MatchDirectories[idx].Message = secPolicy.Spec.Message
}
}
if len(dir.Action) == 0 {
if len(secPolicy.Spec.Process.Action) > 0 {
secPolicy.Spec.Process.MatchDirectories[idx].Action = secPolicy.Spec.Process.Action
} else {
secPolicy.Spec.Process.MatchDirectories[idx].Action = secPolicy.Spec.Action
}
}
}
}
if len(secPolicy.Spec.Process.MatchPatterns) > 0 {
for idx, pat := range secPolicy.Spec.Process.MatchPatterns {
if pat.Severity == 0 {
if secPolicy.Spec.Process.Severity != 0 {
secPolicy.Spec.Process.MatchPatterns[idx].Severity = secPolicy.Spec.Process.Severity
} else {
secPolicy.Spec.Process.MatchPatterns[idx].Severity = secPolicy.Spec.Severity
}
}
if len(pat.Tags) == 0 {
if len(secPolicy.Spec.Process.Tags) > 0 {
secPolicy.Spec.Process.MatchPatterns[idx].Tags = secPolicy.Spec.Process.Tags
} else {
secPolicy.Spec.Process.MatchPatterns[idx].Tags = secPolicy.Spec.Tags
}
}
if len(pat.Message) == 0 {
if len(secPolicy.Spec.Process.Message) > 0 {
secPolicy.Spec.Process.MatchPatterns[idx].Message = secPolicy.Spec.Process.Message
} else {
secPolicy.Spec.Process.MatchPatterns[idx].Message = secPolicy.Spec.Message
}
}
if len(pat.Action) == 0 {
if len(secPolicy.Spec.Process.Action) > 0 {
secPolicy.Spec.Process.MatchPatterns[idx].Action = secPolicy.Spec.Process.Action
} else {
secPolicy.Spec.Process.MatchPatterns[idx].Action = secPolicy.Spec.Action
}
}
}
}
if len(secPolicy.Spec.File.MatchPaths) > 0 {
for idx, path := range secPolicy.Spec.File.MatchPaths {
if path.Severity == 0 {
if secPolicy.Spec.File.Severity != 0 {
secPolicy.Spec.File.MatchPaths[idx].Severity = secPolicy.Spec.File.Severity
} else {
secPolicy.Spec.File.MatchPaths[idx].Severity = secPolicy.Spec.Severity
}
}
if len(path.Tags) == 0 {
if len(secPolicy.Spec.File.Tags) > 0 {
secPolicy.Spec.File.MatchPaths[idx].Tags = secPolicy.Spec.File.Tags
} else {
secPolicy.Spec.File.MatchPaths[idx].Tags = secPolicy.Spec.Tags
}
}
if len(path.Message) == 0 {
if len(secPolicy.Spec.File.Message) > 0 {
secPolicy.Spec.File.MatchPaths[idx].Message = secPolicy.Spec.File.Message
} else {
secPolicy.Spec.File.MatchPaths[idx].Message = secPolicy.Spec.Message
}
}
if len(path.Action) == 0 {
if len(secPolicy.Spec.File.Action) > 0 {
secPolicy.Spec.File.MatchPaths[idx].Action = secPolicy.Spec.File.Action
} else {
secPolicy.Spec.File.MatchPaths[idx].Action = secPolicy.Spec.Action
}
}
}
}
if len(secPolicy.Spec.File.MatchDirectories) > 0 {
for idx, dir := range secPolicy.Spec.File.MatchDirectories {
if dir.Severity == 0 {
if secPolicy.Spec.File.Severity != 0 {
secPolicy.Spec.File.MatchDirectories[idx].Severity = secPolicy.Spec.File.Severity
} else {
secPolicy.Spec.File.MatchDirectories[idx].Severity = secPolicy.Spec.Severity
}
}
if len(dir.Tags) == 0 {
if len(secPolicy.Spec.File.Tags) > 0 {
secPolicy.Spec.File.MatchDirectories[idx].Tags = secPolicy.Spec.File.Tags
} else {
secPolicy.Spec.File.MatchDirectories[idx].Tags = secPolicy.Spec.Tags
}
}
if len(dir.Message) == 0 {
if len(secPolicy.Spec.File.Message) > 0 {
secPolicy.Spec.File.MatchDirectories[idx].Message = secPolicy.Spec.File.Message
} else {
secPolicy.Spec.File.MatchDirectories[idx].Message = secPolicy.Spec.Message
}
}
if len(dir.Action) == 0 {
if len(secPolicy.Spec.File.Action) > 0 {
secPolicy.Spec.File.MatchDirectories[idx].Action = secPolicy.Spec.File.Action
} else {
secPolicy.Spec.File.MatchDirectories[idx].Action = secPolicy.Spec.Action
}
}
}
}
if len(secPolicy.Spec.File.MatchPatterns) > 0 {
for idx, pat := range secPolicy.Spec.File.MatchPatterns {
if pat.Severity == 0 {
if secPolicy.Spec.File.Severity != 0 {
secPolicy.Spec.File.MatchPatterns[idx].Severity = secPolicy.Spec.File.Severity
} else {
secPolicy.Spec.File.MatchPatterns[idx].Severity = secPolicy.Spec.Severity
}
}
if len(pat.Tags) == 0 {
if len(secPolicy.Spec.File.Tags) > 0 {
secPolicy.Spec.File.MatchPatterns[idx].Tags = secPolicy.Spec.File.Tags
} else {
secPolicy.Spec.File.MatchPatterns[idx].Tags = secPolicy.Spec.Tags
}
}
if len(pat.Message) == 0 {
if len(secPolicy.Spec.File.Message) > 0 {
secPolicy.Spec.File.MatchPatterns[idx].Message = secPolicy.Spec.File.Message
} else {
secPolicy.Spec.File.MatchPatterns[idx].Message = secPolicy.Spec.Message
}
}
if len(pat.Action) == 0 {
if len(secPolicy.Spec.File.Action) > 0 {
secPolicy.Spec.File.MatchPatterns[idx].Action = secPolicy.Spec.File.Action
} else {
secPolicy.Spec.File.MatchPatterns[idx].Action = secPolicy.Spec.Action
}
}
}
}
if len(secPolicy.Spec.Network.MatchProtocols) > 0 {
for idx, proto := range secPolicy.Spec.Network.MatchProtocols {
if proto.Severity == 0 {
if secPolicy.Spec.Network.Severity != 0 {
secPolicy.Spec.Network.MatchProtocols[idx].Severity = secPolicy.Spec.Network.Severity
} else {
secPolicy.Spec.Network.MatchProtocols[idx].Severity = secPolicy.Spec.Severity
}
}
if len(proto.Tags) == 0 {
if len(secPolicy.Spec.Network.Tags) > 0 {
secPolicy.Spec.Network.MatchProtocols[idx].Tags = secPolicy.Spec.Network.Tags
} else {
secPolicy.Spec.Network.MatchProtocols[idx].Tags = secPolicy.Spec.Tags
}
}
if len(proto.Message) == 0 {
if len(secPolicy.Spec.Network.Message) > 0 {
secPolicy.Spec.Network.MatchProtocols[idx].Message = secPolicy.Spec.Network.Message
} else {
secPolicy.Spec.Network.MatchProtocols[idx].Message = secPolicy.Spec.Message
}
}
if len(proto.Action) == 0 {
if len(secPolicy.Spec.Network.Action) > 0 {
secPolicy.Spec.Network.MatchProtocols[idx].Action = secPolicy.Spec.Network.Action
} else {
secPolicy.Spec.Network.MatchProtocols[idx].Action = secPolicy.Spec.Action
}
}
}
}
if len(secPolicy.Spec.Capabilities.MatchCapabilities) > 0 {
for idx, cap := range secPolicy.Spec.Capabilities.MatchCapabilities {
if cap.Severity == 0 {
if secPolicy.Spec.Capabilities.Severity != 0 {
secPolicy.Spec.Capabilities.MatchCapabilities[idx].Severity = secPolicy.Spec.Capabilities.Severity
} else {
secPolicy.Spec.Capabilities.MatchCapabilities[idx].Severity = secPolicy.Spec.Severity
}
}
if len(cap.Tags) == 0 {
if len(secPolicy.Spec.Capabilities.Tags) > 0 {
secPolicy.Spec.Capabilities.MatchCapabilities[idx].Tags = secPolicy.Spec.Capabilities.Tags
} else {
secPolicy.Spec.Capabilities.MatchCapabilities[idx].Tags = secPolicy.Spec.Tags
}
}
if len(cap.Message) == 0 {
if len(secPolicy.Spec.Capabilities.Message) > 0 {
secPolicy.Spec.Capabilities.MatchCapabilities[idx].Message = secPolicy.Spec.Capabilities.Message
} else {
secPolicy.Spec.Capabilities.MatchCapabilities[idx].Message = secPolicy.Spec.Message
}
}
if len(cap.Action) == 0 {
if len(secPolicy.Spec.Capabilities.Action) > 0 {
secPolicy.Spec.Capabilities.MatchCapabilities[idx].Action = secPolicy.Spec.Capabilities.Action
} else {
secPolicy.Spec.Capabilities.MatchCapabilities[idx].Action = secPolicy.Spec.Action
}
}
}
}
if len(secPolicy.Spec.Syscalls.MatchSyscalls) > 0 {
for idx, syscall := range secPolicy.Spec.Syscalls.MatchSyscalls {
if syscall.Severity == 0 {
if secPolicy.Spec.Syscalls.Severity != 0 {
secPolicy.Spec.Syscalls.MatchSyscalls[idx].Severity = secPolicy.Spec.Syscalls.Severity
} else {
secPolicy.Spec.Syscalls.MatchSyscalls[idx].Severity = secPolicy.Spec.Severity
}
}
if len(syscall.Tags) == 0 {
if len(secPolicy.Spec.Syscalls.Tags) > 0 {
secPolicy.Spec.Syscalls.MatchSyscalls[idx].Tags = secPolicy.Spec.Syscalls.Tags
} else {
secPolicy.Spec.Syscalls.MatchSyscalls[idx].Tags = secPolicy.Spec.Tags
}
}
if len(syscall.Message) == 0 {
if len(secPolicy.Spec.Syscalls.Message) > 0 {
secPolicy.Spec.Syscalls.MatchSyscalls[idx].Message = secPolicy.Spec.Syscalls.Message
} else {
secPolicy.Spec.Syscalls.MatchSyscalls[idx].Message = secPolicy.Spec.Message
}
}
}
}
if len(secPolicy.Spec.Syscalls.MatchPaths) > 0 {
for idx, syscall := range secPolicy.Spec.Syscalls.MatchPaths {
if syscall.Severity == 0 {
if secPolicy.Spec.Syscalls.Severity != 0 {
secPolicy.Spec.Syscalls.MatchPaths[idx].Severity = secPolicy.Spec.Syscalls.Severity
} else {
secPolicy.Spec.Syscalls.MatchPaths[idx].Severity = secPolicy.Spec.Severity
}
}
if len(syscall.Tags) == 0 {
if len(secPolicy.Spec.Syscalls.Tags) > 0 {
secPolicy.Spec.Syscalls.MatchPaths[idx].Tags = secPolicy.Spec.Syscalls.Tags
} else {
secPolicy.Spec.Syscalls.MatchPaths[idx].Tags = secPolicy.Spec.Tags
}
}
if len(syscall.Message) == 0 {
if len(secPolicy.Spec.Syscalls.Message) > 0 {
secPolicy.Spec.Syscalls.MatchPaths[idx].Message = secPolicy.Spec.Syscalls.Message
} else {
secPolicy.Spec.Syscalls.MatchPaths[idx].Message = secPolicy.Spec.Message
}
}
}
}
return
}
// WatchSecurityPolicies Function
func (dm *KubeArmorDaemon) WatchSecurityPolicies() cache.InformerSynced {
for {
if !K8s.CheckCustomResourceDefinition("kubearmorpolicies") {
time.Sleep(time.Second * 1)
continue
} else {
break
}
}
factory := kspinformer.NewSharedInformerFactory(K8s.KSPClient, 0)
informer := factory.Security().V1().KubeArmorPolicies().Informer()
registration, err := informer.AddEventHandler(
cache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
// create a security policy
if policy, ok := obj.(*ksp.KubeArmorPolicy); ok {
secPolicy, err := dm.CreateSecurityPolicy(KubeArmorPolicy, *policy)
if err != nil {
dm.Logger.Warnf("Error ADD, %s", err)
return
}
dm.SecurityPoliciesLock.Lock()
new := true
for _, policy := range dm.SecurityPolicies {
if policy.Metadata["namespaceName"] == secPolicy.Metadata["namespaceName"] && policy.Metadata["policyName"] == secPolicy.Metadata["policyName"] {
new = false
break
}
}
if new {
dm.SecurityPolicies = append(dm.SecurityPolicies, secPolicy)
}
dm.SecurityPoliciesLock.Unlock()
dm.Logger.Printf("Detected a Security Policy (added/%s/%s)", secPolicy.Metadata["namespaceName"], secPolicy.Metadata["policyName"])
// apply security policies to pods
dm.UpdateSecurityPolicy(addEvent, KubeArmorPolicy, secPolicy)
}
},
UpdateFunc: func(oldObj, newObj interface{}) {
if policy, ok := newObj.(*ksp.KubeArmorPolicy); ok {
secPolicy, err := dm.CreateSecurityPolicy(KubeArmorPolicy, *policy)
if err != nil {
return
}
dm.SecurityPoliciesLock.Lock()
for idx, policy := range dm.SecurityPolicies {
if policy.Metadata["namespaceName"] == secPolicy.Metadata["namespaceName"] && policy.Metadata["policyName"] == secPolicy.Metadata["policyName"] {
dm.SecurityPolicies[idx] = secPolicy
break
}
}
dm.SecurityPoliciesLock.Unlock()
dm.Logger.Printf("Detected a Security Policy (modified/%s/%s)", secPolicy.Metadata["namespaceName"], secPolicy.Metadata["policyName"])
// apply security policies to pods
dm.UpdateSecurityPolicy(updateEvent, KubeArmorPolicy, secPolicy)
}
},
DeleteFunc: func(obj interface{}) {
if policy, ok := obj.(*ksp.KubeArmorPolicy); ok {
secPolicy, err := dm.CreateSecurityPolicy(KubeArmorPolicy, *policy)
if err != nil {
return
}
dm.SecurityPoliciesLock.Lock()
for idx, policy := range dm.SecurityPolicies {
if policy.Metadata["namespaceName"] == secPolicy.Metadata["namespaceName"] && policy.Metadata["policyName"] == secPolicy.Metadata["policyName"] {
dm.SecurityPolicies = append(dm.SecurityPolicies[:idx], dm.SecurityPolicies[idx+1:]...)
break
}
}
dm.SecurityPoliciesLock.Unlock()
dm.Logger.Printf("Detected a Security Policy (deleted/%s/%s)", secPolicy.Metadata["namespaceName"], secPolicy.Metadata["policyName"])
// apply security policies to pods
dm.UpdateSecurityPolicy(deleteEvent, KubeArmorPolicy, secPolicy)
}
},
},
)
if err != nil {
dm.Logger.Err("Couldn't start watching KubeArmor Security Policies")
return nil
}
go factory.Start(StopChan)
return registration.HasSynced
}
// WatchClusterSecurityPolicies Function
func (dm *KubeArmorDaemon) WatchClusterSecurityPolicies(timeout time.Duration) cache.InformerSynced {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
crdFound := false
for !crdFound {
select {
case <-ctx.Done():
dm.Logger.Warn("timeout while monitoring cluster security policies, kubearmorclusterpolicies CRD not found")
return nil
default:
if K8s.CheckCustomResourceDefinition("kubearmorclusterpolicies") {
crdFound = true
} else {
time.Sleep(time.Second * 1)
}
}
}
factory := kspinformer.NewSharedInformerFactory(K8s.KSPClient, 0)
informer := factory.Security().V1().KubeArmorClusterPolicies().Informer()
registration, err := informer.AddEventHandler(
cache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
// create a security policy
if policy, ok := obj.(*ksp.KubeArmorClusterPolicy); ok {
secPolicy, err := dm.CreateSecurityPolicy(KubeArmorClusterPolicy, *policy)
if err != nil {
dm.Logger.Warnf("Error ADD, %s", err)
return
}
dm.SecurityPoliciesLock.Lock()
new := true
for _, policy := range dm.SecurityPolicies {
if policy.Metadata["policyName"] == secPolicy.Metadata["policyName"] {
new = false
break
}
}
if new {
dm.SecurityPolicies = append(dm.SecurityPolicies, secPolicy)
}
dm.SecurityPoliciesLock.Unlock()
dm.Logger.Printf("Detected a Cluster Security Policy (added/%s)", secPolicy.Metadata["policyName"])
// apply security policies to pods
dm.UpdateSecurityPolicy(addEvent, KubeArmorClusterPolicy, secPolicy)
}
},
UpdateFunc: func(oldObj, newObj interface{}) {
if policy, ok := newObj.(*ksp.KubeArmorClusterPolicy); ok {
secPolicy, err := dm.CreateSecurityPolicy(KubeArmorClusterPolicy, *policy)
if err != nil {
return
}
dm.SecurityPoliciesLock.Lock()
for idx, policy := range dm.SecurityPolicies {
if policy.Metadata["policyName"] == secPolicy.Metadata["policyName"] {
dm.SecurityPolicies[idx] = secPolicy
break
}
}
dm.SecurityPoliciesLock.Unlock()
dm.Logger.Printf("Detected a Cluster Security Policy (modified/%s)", secPolicy.Metadata["policyName"])
// apply security policies to pods
dm.UpdateSecurityPolicy(updateEvent, KubeArmorClusterPolicy, secPolicy)
}
},
DeleteFunc: func(obj interface{}) {
if policy, ok := obj.(*ksp.KubeArmorClusterPolicy); ok {
secPolicy, err := dm.CreateSecurityPolicy(KubeArmorClusterPolicy, *policy)
if err != nil {
return
}
dm.SecurityPoliciesLock.Lock()
for idx, policy := range dm.SecurityPolicies {
if policy.Metadata["policyName"] == secPolicy.Metadata["policyName"] {
dm.SecurityPolicies = append(dm.SecurityPolicies[:idx], dm.SecurityPolicies[idx+1:]...)
break
}
}
dm.SecurityPoliciesLock.Unlock()
dm.Logger.Printf("Detected a Cluster Security Policy (deleted/%s)", secPolicy.Metadata["policyName"])
// apply security policies to pods
dm.UpdateSecurityPolicy(deleteEvent, KubeArmorClusterPolicy, secPolicy)
}
},
},
)
if err != nil {
dm.Logger.Err("Couldn't start watching KubeArmor Security Policies")
return nil
}
go factory.Start(StopChan)
return registration.HasSynced
}
// ================================= //
// == Host Security Policy Update == //
// ================================= //
// UpdateHostSecurityPolicies Function
func (dm *KubeArmorDaemon) UpdateHostSecurityPolicies() {
dm.HostSecurityPoliciesLock.RLock()
hostSecurityPoliciesLength := len(dm.HostSecurityPolicies)
dm.HostSecurityPoliciesLock.RUnlock()
secPolicies := []tp.HostSecurityPolicy{}
for idx := 0; idx < hostSecurityPoliciesLength; idx++ {
dm.EndPointsLock.RLock()
policy := dm.HostSecurityPolicies[idx]
dm.EndPointsLock.RUnlock()
if kl.MatchIdentities(policy.Spec.NodeSelector.Identities, dm.Node.Identities) {
secPolicies = append(secPolicies, policy)
}
}
if cfg.GlobalCfg.HostPolicy {
// update host security policies
dm.Logger.UpdateHostSecurityPolicies("UPDATED", secPolicies)
if dm.RuntimeEnforcer != nil {
if dm.Node.PolicyEnabled == tp.KubeArmorPolicyEnabled {
// enforce host security policies
dm.RuntimeEnforcer.UpdateHostSecurityPolicies(secPolicies)
}
}
}
}
// ParseAndUpdateHostSecurityPolicy Function
func (dm *KubeArmorDaemon) ParseAndUpdateHostSecurityPolicy(event tp.K8sKubeArmorHostPolicyEvent) pb.PolicyStatus {
// create a host security policy
secPolicy := tp.HostSecurityPolicy{}
secPolicy.Metadata = map[string]string{}
secPolicy.Metadata["policyName"] = event.Object.Metadata.Name
if err := kl.Clone(event.Object.Spec, &secPolicy.Spec); err != nil {
dm.Logger.Errf("Failed to clone a spec (%s)", err.Error())
return pb.PolicyStatus_Failure
}
kl.ObjCommaExpandFirstDupOthers(&secPolicy.Spec.Network.MatchProtocols)
kl.ObjCommaExpandFirstDupOthers(&secPolicy.Spec.Capabilities.MatchCapabilities)
if secPolicy.Spec.Severity == 0 {
secPolicy.Spec.Severity = 1 // the lowest severity, by default
}
switch secPolicy.Spec.Action {
case "allow":
secPolicy.Spec.Action = "Allow"
case "audit":
secPolicy.Spec.Action = "Audit"
case "block":
secPolicy.Spec.Action = "Block"
case "":
secPolicy.Spec.Action = "Block" // by default
}
// add identities
secPolicy.Spec.NodeSelector.Identities = []string{}
for k, v := range secPolicy.Spec.NodeSelector.MatchLabels {
secPolicy.Spec.NodeSelector.Identities = append(secPolicy.Spec.NodeSelector.Identities, k+"="+v)
}
sort.Slice(secPolicy.Spec.NodeSelector.Identities, func(i, j int) bool {
return secPolicy.Spec.NodeSelector.Identities[i] < secPolicy.Spec.NodeSelector.Identities[j]
})
// add severities, tags, messages, and actions
if len(secPolicy.Spec.Process.MatchPaths) > 0 {
for idx, path := range secPolicy.Spec.Process.MatchPaths {
if path.Severity == 0 {
if secPolicy.Spec.Process.Severity != 0 {
secPolicy.Spec.Process.MatchPaths[idx].Severity = secPolicy.Spec.Process.Severity
} else {
secPolicy.Spec.Process.MatchPaths[idx].Severity = secPolicy.Spec.Severity
}
}
if len(path.Tags) == 0 {
if len(secPolicy.Spec.Process.Tags) > 0 {
secPolicy.Spec.Process.MatchPaths[idx].Tags = secPolicy.Spec.Process.Tags
} else {
secPolicy.Spec.Process.MatchPaths[idx].Tags = secPolicy.Spec.Tags
}
}
if len(path.Message) == 0 {
if len(secPolicy.Spec.Process.Message) > 0 {
secPolicy.Spec.Process.MatchPaths[idx].Message = secPolicy.Spec.Process.Message
} else {
secPolicy.Spec.Process.MatchPaths[idx].Message = secPolicy.Spec.Message
}
}
if len(path.Action) == 0 {
if len(secPolicy.Spec.Process.Action) > 0 {
secPolicy.Spec.Process.MatchPaths[idx].Action = secPolicy.Spec.Process.Action
} else {
secPolicy.Spec.Process.MatchPaths[idx].Action = secPolicy.Spec.Action
}
}
}
}
if len(secPolicy.Spec.Process.MatchDirectories) > 0 {
for idx, dir := range secPolicy.Spec.Process.MatchDirectories {
if dir.Severity == 0 {
if secPolicy.Spec.Process.Severity != 0 {
secPolicy.Spec.Process.MatchDirectories[idx].Severity = secPolicy.Spec.Process.Severity
} else {
secPolicy.Spec.Process.MatchDirectories[idx].Severity = secPolicy.Spec.Severity
}
}
if len(dir.Tags) == 0 {
if len(secPolicy.Spec.Process.Tags) > 0 {
secPolicy.Spec.Process.MatchDirectories[idx].Tags = secPolicy.Spec.Process.Tags
} else {
secPolicy.Spec.Process.MatchDirectories[idx].Tags = secPolicy.Spec.Tags
}
}
if len(dir.Message) == 0 {
if len(secPolicy.Spec.Process.Message) > 0 {
secPolicy.Spec.Process.MatchDirectories[idx].Message = secPolicy.Spec.Process.Message
} else {
secPolicy.Spec.Process.MatchDirectories[idx].Message = secPolicy.Spec.Message
}
}
if len(dir.Action) == 0 {
if len(secPolicy.Spec.Process.Action) > 0 {
secPolicy.Spec.Process.MatchDirectories[idx].Action = secPolicy.Spec.Process.Action
} else {
secPolicy.Spec.Process.MatchDirectories[idx].Action = secPolicy.Spec.Action
}
}
}
}
if len(secPolicy.Spec.Process.MatchPatterns) > 0 {
for idx, pat := range secPolicy.Spec.Process.MatchPatterns {
if pat.Severity == 0 {
if secPolicy.Spec.Process.Severity != 0 {
secPolicy.Spec.Process.MatchPatterns[idx].Severity = secPolicy.Spec.Process.Severity
} else {
secPolicy.Spec.Process.MatchPatterns[idx].Severity = secPolicy.Spec.Severity
}
}
if len(pat.Tags) == 0 {
if len(secPolicy.Spec.Process.Tags) > 0 {
secPolicy.Spec.Process.MatchPatterns[idx].Tags = secPolicy.Spec.Process.Tags
} else {
secPolicy.Spec.Process.MatchPatterns[idx].Tags = secPolicy.Spec.Tags
}
}
if len(pat.Message) == 0 {
if len(secPolicy.Spec.Process.Message) > 0 {
secPolicy.Spec.Process.MatchPatterns[idx].Message = secPolicy.Spec.Process.Message
} else {
secPolicy.Spec.Process.MatchPatterns[idx].Message = secPolicy.Spec.Message
}
}
if len(pat.Action) == 0 {
if len(secPolicy.Spec.Process.Action) > 0 {
secPolicy.Spec.Process.MatchPatterns[idx].Action = secPolicy.Spec.Process.Action
} else {
secPolicy.Spec.Process.MatchPatterns[idx].Action = secPolicy.Spec.Action
}
}
}
}
if len(secPolicy.Spec.File.MatchPaths) > 0 {
for idx, path := range secPolicy.Spec.File.MatchPaths {
if path.Severity == 0 {
if secPolicy.Spec.File.Severity != 0 {
secPolicy.Spec.File.MatchPaths[idx].Severity = secPolicy.Spec.File.Severity
} else {
secPolicy.Spec.File.MatchPaths[idx].Severity = secPolicy.Spec.Severity
}
}
if len(path.Tags) == 0 {
if len(secPolicy.Spec.File.Tags) > 0 {
secPolicy.Spec.File.MatchPaths[idx].Tags = secPolicy.Spec.File.Tags
} else {
secPolicy.Spec.File.MatchPaths[idx].Tags = secPolicy.Spec.Tags
}
}
if len(path.Message) == 0 {
if len(secPolicy.Spec.File.Message) > 0 {
secPolicy.Spec.File.MatchPaths[idx].Message = secPolicy.Spec.File.Message
} else {
secPolicy.Spec.File.MatchPaths[idx].Message = secPolicy.Spec.Message
}
}
if len(path.Action) == 0 {
if len(secPolicy.Spec.File.Action) > 0 {
secPolicy.Spec.File.MatchPaths[idx].Action = secPolicy.Spec.File.Action
} else {
secPolicy.Spec.File.MatchPaths[idx].Action = secPolicy.Spec.Action
}
}
}
}
if len(secPolicy.Spec.File.MatchDirectories) > 0 {
for idx, dir := range secPolicy.Spec.File.MatchDirectories {
if dir.Severity == 0 {
if secPolicy.Spec.File.Severity != 0 {
secPolicy.Spec.File.MatchDirectories[idx].Severity = secPolicy.Spec.File.Severity
} else {
secPolicy.Spec.File.MatchDirectories[idx].Severity = secPolicy.Spec.Severity
}
}
if len(dir.Tags) == 0 {
if len(secPolicy.Spec.File.Tags) > 0 {
secPolicy.Spec.File.MatchDirectories[idx].Tags = secPolicy.Spec.File.Tags
} else {
secPolicy.Spec.File.MatchDirectories[idx].Tags = secPolicy.Spec.Tags
}
}
if len(dir.Message) == 0 {
if len(secPolicy.Spec.File.Message) > 0 {
secPolicy.Spec.File.MatchDirectories[idx].Message = secPolicy.Spec.File.Message
} else {
secPolicy.Spec.File.MatchDirectories[idx].Message = secPolicy.Spec.Message
}
}
if len(dir.Action) == 0 {
if len(secPolicy.Spec.File.Action) > 0 {
secPolicy.Spec.File.MatchDirectories[idx].Action = secPolicy.Spec.File.Action
} else {
secPolicy.Spec.File.MatchDirectories[idx].Action = secPolicy.Spec.Action
}
}
}
}
if len(secPolicy.Spec.File.MatchPatterns) > 0 {
for idx, pat := range secPolicy.Spec.File.MatchPatterns {
if pat.Severity == 0 {
if secPolicy.Spec.File.Severity != 0 {
secPolicy.Spec.File.MatchPatterns[idx].Severity = secPolicy.Spec.File.Severity
} else {
secPolicy.Spec.File.MatchPatterns[idx].Severity = secPolicy.Spec.Severity
}
}
if len(pat.Tags) == 0 {
if len(secPolicy.Spec.File.Tags) > 0 {
secPolicy.Spec.File.MatchPatterns[idx].Tags = secPolicy.Spec.File.Tags
} else {
secPolicy.Spec.File.MatchPatterns[idx].Tags = secPolicy.Spec.Tags
}
}
if len(pat.Message) == 0 {
if len(secPolicy.Spec.File.Message) > 0 {
secPolicy.Spec.File.MatchPatterns[idx].Message = secPolicy.Spec.File.Message
} else {
secPolicy.Spec.File.MatchPatterns[idx].Message = secPolicy.Spec.Message
}
}
if len(pat.Action) == 0 {
if len(secPolicy.Spec.File.Action) > 0 {
secPolicy.Spec.File.MatchPatterns[idx].Action = secPolicy.Spec.File.Action
} else {
secPolicy.Spec.File.MatchPatterns[idx].Action = secPolicy.Spec.Action
}
}
}
}
if len(secPolicy.Spec.Network.MatchProtocols) > 0 {
for idx, proto := range secPolicy.Spec.Network.MatchProtocols {
if proto.Severity == 0 {
if secPolicy.Spec.Network.Severity != 0 {
secPolicy.Spec.Network.MatchProtocols[idx].Severity = secPolicy.Spec.Network.Severity
} else {
secPolicy.Spec.Network.MatchProtocols[idx].Severity = secPolicy.Spec.Severity
}
}
if len(proto.Tags) == 0 {
if len(secPolicy.Spec.Network.Tags) > 0 {
secPolicy.Spec.Network.MatchProtocols[idx].Tags = secPolicy.Spec.Network.Tags
} else {
secPolicy.Spec.Network.MatchProtocols[idx].Tags = secPolicy.Spec.Tags
}
}
if len(proto.Message) == 0 {
if len(secPolicy.Spec.Network.Message) > 0 {
secPolicy.Spec.Network.MatchProtocols[idx].Message = secPolicy.Spec.Network.Message
} else {
secPolicy.Spec.Network.MatchProtocols[idx].Message = secPolicy.Spec.Message
}
}
if len(proto.Action) == 0 {
if len(secPolicy.Spec.Network.Action) > 0 {
secPolicy.Spec.Network.MatchProtocols[idx].Action = secPolicy.Spec.Network.Action
} else {
secPolicy.Spec.Network.MatchProtocols[idx].Action = secPolicy.Spec.Action
}
}
}
}
if len(secPolicy.Spec.Capabilities.MatchCapabilities) > 0 {
for idx, cap := range secPolicy.Spec.Capabilities.MatchCapabilities {
if cap.Severity == 0 {
if secPolicy.Spec.Capabilities.Severity != 0 {
secPolicy.Spec.Capabilities.MatchCapabilities[idx].Severity = secPolicy.Spec.Capabilities.Severity
} else {
secPolicy.Spec.Capabilities.MatchCapabilities[idx].Severity = secPolicy.Spec.Severity
}
}
if len(cap.Tags) == 0 {
if len(secPolicy.Spec.Capabilities.Tags) > 0 {
secPolicy.Spec.Capabilities.MatchCapabilities[idx].Tags = secPolicy.Spec.Capabilities.Tags
} else {
secPolicy.Spec.Capabilities.MatchCapabilities[idx].Tags = secPolicy.Spec.Tags
}
}
if len(cap.Message) == 0 {
if len(secPolicy.Spec.Capabilities.Message) > 0 {
secPolicy.Spec.Capabilities.MatchCapabilities[idx].Message = secPolicy.Spec.Capabilities.Message
} else {
secPolicy.Spec.Capabilities.MatchCapabilities[idx].Message = secPolicy.Spec.Message
}
}
if len(cap.Action) == 0 {
if len(secPolicy.Spec.Capabilities.Action) > 0 {
secPolicy.Spec.Capabilities.MatchCapabilities[idx].Action = secPolicy.Spec.Capabilities.Action
} else {
secPolicy.Spec.Capabilities.MatchCapabilities[idx].Action = secPolicy.Spec.Action
}
}
}
}
if len(secPolicy.Spec.Syscalls.MatchSyscalls) > 0 {
for idx, syscall := range secPolicy.Spec.Syscalls.MatchSyscalls {
if syscall.Severity == 0 {
if secPolicy.Spec.Syscalls.Severity != 0 {
secPolicy.Spec.Syscalls.MatchSyscalls[idx].Severity = secPolicy.Spec.Syscalls.Severity
} else {
secPolicy.Spec.Syscalls.MatchSyscalls[idx].Severity = secPolicy.Spec.Severity
}
}
if len(syscall.Tags) == 0 {
if len(secPolicy.Spec.Syscalls.Tags) > 0 {
secPolicy.Spec.Syscalls.MatchSyscalls[idx].Tags = secPolicy.Spec.Syscalls.Tags
} else {
secPolicy.Spec.Syscalls.MatchSyscalls[idx].Tags = secPolicy.Spec.Tags
}
}
if len(syscall.Message) == 0 {
if len(secPolicy.Spec.Syscalls.Message) > 0 {
secPolicy.Spec.Syscalls.MatchSyscalls[idx].Message = secPolicy.Spec.Syscalls.Message
} else {
secPolicy.Spec.Syscalls.MatchSyscalls[idx].Message = secPolicy.Spec.Message
}
}
}
}
if len(secPolicy.Spec.Syscalls.MatchPaths) > 0 {
for idx, syscall := range secPolicy.Spec.Syscalls.MatchPaths {
if syscall.Severity == 0 {
if secPolicy.Spec.Syscalls.Severity != 0 {
secPolicy.Spec.Syscalls.MatchPaths[idx].Severity = secPolicy.Spec.Syscalls.Severity
} else {
secPolicy.Spec.Syscalls.MatchPaths[idx].Severity = secPolicy.Spec.Severity
}
}
if len(syscall.Tags) == 0 {
if len(secPolicy.Spec.Syscalls.Tags) > 0 {
secPolicy.Spec.Syscalls.MatchPaths[idx].Tags = secPolicy.Spec.Syscalls.Tags
} else {
secPolicy.Spec.Syscalls.MatchPaths[idx].Tags = secPolicy.Spec.Tags
}
}
if len(syscall.Message) == 0 {
if len(secPolicy.Spec.Syscalls.Message) > 0 {
secPolicy.Spec.Syscalls.MatchPaths[idx].Message = secPolicy.Spec.Syscalls.Message
} else {
secPolicy.Spec.Syscalls.MatchPaths[idx].Message = secPolicy.Spec.Message
}
}
}
}
// update a security policy into the policy list
dm.HostSecurityPoliciesLock.Lock()
if event.Type == addEvent {
new := true
for idx, policy := range dm.HostSecurityPolicies {
if policy.Metadata["policyName"] == secPolicy.Metadata["policyName"] {
if reflect.DeepEqual(policy, secPolicy) {
kg.Debugf("No updates to policy %s", policy.Metadata["policyName"])
dm.HostSecurityPoliciesLock.Unlock()
return pb.PolicyStatus_Applied
}
dm.HostSecurityPolicies[idx] = secPolicy
event.Type = updateEvent
new = false
break
}
}
if new {
dm.HostSecurityPolicies = append(dm.HostSecurityPolicies, secPolicy)
}
} else if event.Type == updateEvent {
for idx, policy := range dm.HostSecurityPolicies {
if policy.Metadata["policyName"] == secPolicy.Metadata["policyName"] {
if reflect.DeepEqual(policy, secPolicy) {
kg.Debugf("No updates to policy %s", policy.Metadata["policyName"])
dm.HostSecurityPoliciesLock.Unlock()
return pb.PolicyStatus_Applied
}
dm.HostSecurityPolicies[idx] = secPolicy
break
}
}
} else if event.Type == deleteEvent {
// check that a security policy should exist before performing delete operation
policymatch := false
for idx, policy := range dm.HostSecurityPolicies {
if policy.Metadata["policyName"] == secPolicy.Metadata["policyName"] {
dm.HostSecurityPolicies = append(dm.HostSecurityPolicies[:idx], dm.HostSecurityPolicies[idx+1:]...)
policymatch = true
break
}
}
if !policymatch {
dm.Logger.Warnf("Failed to delete security policy. Policy doesn't exist")
dm.HostSecurityPoliciesLock.Unlock()
return pb.PolicyStatus_NotExist
}
}
dm.HostSecurityPoliciesLock.Unlock()
dm.Logger.Printf("Detected a Host Security Policy (%s/%s)", strings.ToLower(event.Type), secPolicy.Metadata["policyName"])
// apply security policies to a host
dm.UpdateHostSecurityPolicies()
if !cfg.GlobalCfg.K8sEnv && (cfg.GlobalCfg.KVMAgent || cfg.GlobalCfg.HostPolicy) {
if event.Type == addEvent || event.Type == updateEvent {
// backup HostSecurityPolicy to file
dm.backupKubeArmorHostPolicy(secPolicy)
} else if event.Type == deleteEvent {
dm.removeBackUpPolicy(secPolicy.Metadata["policyName"])
}
}
if event.Type == addEvent {
return pb.PolicyStatus_Applied
} else if event.Type == deleteEvent {
return pb.PolicyStatus_Deleted
}
return pb.PolicyStatus_Modified
}
// WatchHostSecurityPolicies Function
func (dm *KubeArmorDaemon) WatchHostSecurityPolicies(timeout time.Duration) {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
for {
select {
case <-ctx.Done():
dm.Logger.Warn("timeout while monitoring host security policies, kubearmorhostpolicies CRD not found")
return
default:
if !K8s.CheckCustomResourceDefinition("kubearmorhostpolicies") {
time.Sleep(time.Second * 1)
continue
}
}
dm.Logger.Print("Started to monitor host security policies")
if !K8s.CheckCustomResourceDefinition("kubearmorhostpolicies") {
time.Sleep(time.Second * 1)
continue
}
if resp := K8s.WatchK8sHostSecurityPolicies(); resp != nil {
defer func() {
if err := resp.Body.Close(); err != nil {
kg.Warnf("Error closing http stream %s\n", err)
}
}()
decoder := json.NewDecoder(resp.Body)
for {
event := tp.K8sKubeArmorHostPolicyEvent{}
if err := decoder.Decode(&event); err == io.EOF {
break
} else if err != nil {
break
}
if event.Object.Status.Status != "" && event.Object.Status.Status != "OK" {
continue
}
if event.Type != addEvent && event.Type != updateEvent && event.Type != deleteEvent {
continue
}
dm.ParseAndUpdateHostSecurityPolicy(event)
}
}
}
}
// ===================== //
// == Default Posture == //
// ===================== //
func (dm *KubeArmorDaemon) updatEndpointsWithCM(cm *corev1.ConfigMap, action string) {
dm.EndPointsLock.Lock()
defer dm.EndPointsLock.Unlock()
dm.DefaultPosturesLock.Lock()
defer dm.DefaultPosturesLock.Unlock()
// get all namespaces
nsList, err := K8s.K8sClient.CoreV1().Namespaces().List(context.Background(), metav1.ListOptions{})
if err != nil {
kg.Err("unable to fetch namespace list")
return
}
// for each namespace if needed change endpoint depfault posture
for _, ns := range nsList.Items {
ns := ns
fp, fa := validateDefaultPosture("kubearmor-file-posture", &ns, cm.Data[cfg.ConfigDefaultFilePosture])
np, na := validateDefaultPosture("kubearmor-network-posture", &ns, cm.Data[cfg.ConfigDefaultNetworkPosture])
cp, ca := validateDefaultPosture("kubearmor-capabilities-posture", &ns, cm.Data[cfg.ConfigDefaultCapabilitiesPosture])
annotated := fa || na || ca // if namespace is annotated for atleast one posture
fullyannotated := fa && na && ca // if namespace is fully annotated
posture := tp.DefaultPosture{
FileAction: fp,
NetworkAction: np,
CapabilitiesAction: cp,
}
// skip if namespace is fully annotated
if fullyannotated {
continue
}
for idx, endpoint := range dm.EndPoints {
// skip all endpoints not in current namespace
if endpoint.NamespaceName != ns.Name {
continue
}
if endpoint.DefaultPosture != posture { // optimization, only if its needed to update the posture
dm.Logger.Printf("updating default posture for %s in %s", ns.Name, endpoint.EndPointName)
dm.UpdateDefaultPostureWithCM(&dm.EndPoints[idx], action, ns.Name, posture, annotated)
}
}
}
}
// UpdateDefaultPostureWithCM Function
func (dm *KubeArmorDaemon) UpdateDefaultPostureWithCM(endPoint *tp.EndPoint, action string, namespace string, defaultPosture tp.DefaultPosture, annotated bool) {
// namespace is (partialy) annotated with posture annotation(s)
if annotated {
// update the dm.DefaultPosture[namespace]
dm.DefaultPostures[namespace] = defaultPosture
}
dm.Logger.UpdateDefaultPosture(action, namespace, defaultPosture)
// update the endpoint with updated default posture
endPoint.DefaultPosture = defaultPosture
dm.Logger.Printf("Updated default posture for %s with %v", endPoint.EndPointName, endPoint.DefaultPosture)
if cfg.GlobalCfg.Policy {
// update security policies
if dm.RuntimeEnforcer != nil {
if endPoint.PolicyEnabled == tp.KubeArmorPolicyEnabled {
// enforce security policies
if !kl.ContainsElement(dm.SystemMonitor.UntrackedNamespaces, endPoint.NamespaceName) {
dm.RuntimeEnforcer.UpdateSecurityPolicies(*endPoint)
} else {
dm.Logger.Warnf("Policy cannot be enforced in untracked namespace %s", endPoint.NamespaceName)
}
}
}
}
}
// returns default posture and a boolean value states, if annotation is set or not
func validateDefaultPosture(key string, ns *corev1.Namespace, defaultPosture string) (string, bool) {
if posture, ok := ns.Annotations[key]; ok {
if posture == "audit" || posture == "Audit" {
return "audit", true
} else if posture == "block" || posture == "Block" {
return "block", true
}
// Invalid Annotation Value, Updating the value to global default
ns.Annotations[key] = defaultPosture
updatedNS, err := K8s.K8sClient.CoreV1().Namespaces().Update(context.Background(), ns, metav1.UpdateOptions{})
if err != nil {
kg.Warnf("Error updating invalid default posture annotation for %v", updatedNS)
}
}
return defaultPosture, false
}
// UpdateDefaultPosture Function
func (dm *KubeArmorDaemon) UpdateDefaultPosture(action string, namespace string, defaultPosture tp.DefaultPosture, annotated bool) {
dm.DefaultPosturesLock.Lock()
defer dm.DefaultPosturesLock.Unlock()
// namespace deleted
if action == deleteEvent {
_, ok := dm.DefaultPostures[namespace]
if ok {
delete(dm.DefaultPostures, namespace)
}
}
// namespace is annotated with posture annotation(s)
if annotated {
dm.DefaultPostures[namespace] = defaultPosture
}
dm.Logger.UpdateDefaultPosture(action, namespace, defaultPosture)
dm.EndPointsLock.RLock()
endPointsLen := len(dm.EndPoints)
dm.EndPointsLock.RUnlock()
for idx := 0; idx < endPointsLen; idx++ {
dm.EndPointsLock.RLock()
endPoint := dm.EndPoints[idx]
dm.EndPointsLock.RUnlock()
// update a security policy
if namespace == endPoint.NamespaceName {
if endPoint.DefaultPosture == defaultPosture {
continue
}
dm.Logger.Printf("Updating default posture for %s with %v namespace default %v", endPoint.EndPointName, endPoint.DefaultPosture, defaultPosture)
endPoint.DefaultPosture = defaultPosture
dm.EndPointsLock.Lock()
dm.EndPoints[idx] = endPoint
dm.EndPointsLock.Unlock()
if cfg.GlobalCfg.Policy {
// update security policies
if dm.RuntimeEnforcer != nil {
if endPoint.PolicyEnabled == tp.KubeArmorPolicyEnabled {
// enforce security policies
if !kl.ContainsElement(dm.SystemMonitor.UntrackedNamespaces, endPoint.NamespaceName) {
dm.RuntimeEnforcer.UpdateSecurityPolicies(endPoint)
} else {
dm.Logger.Warnf("Policy cannot be enforced in untracked namespace %s", endPoint.NamespaceName)
}
}
}
}
}
}
}
func validateGlobalDefaultPosture(posture string) string {
switch posture {
case "audit", "Audit":
return "audit"
case "block", "Block":
return "block"
default:
return "audit"
}
}
// ======================== //
// == Default Visibility == //
// ======================== //
func (dm *KubeArmorDaemon) validateVisibility(scope string, visibility string) bool {
return strings.Contains(visibility, scope)
}
// UpdateVisibility Function
func (dm *KubeArmorDaemon) UpdateVisibility(action string, namespace string, visibility tp.Visibility) {
dm.SystemMonitor.BpfMapLock.Lock()
defer dm.SystemMonitor.BpfMapLock.Unlock()
if action == addEvent || action == updateEvent {
if val, ok := dm.SystemMonitor.NamespacePidsMap[namespace]; ok {
val.Capability = visibility.Capabilities
val.File = visibility.File
val.Network = visibility.Network
val.Process = visibility.Process
val.DNS = visibility.DNS
dm.SystemMonitor.NamespacePidsMap[namespace] = val
for _, nskey := range val.NsKeys {
dm.SystemMonitor.UpdateNsKeyMap(updateEvent, nskey, visibility)
}
} else {
dm.SystemMonitor.NamespacePidsMap[namespace] = monitor.NsVisibility{
NsKeys: []monitor.NsKey{},
File: visibility.File,
Process: visibility.Process,
Capability: visibility.Capabilities,
Network: visibility.Network,
DNS: visibility.DNS,
}
}
dm.Logger.Printf("Namespace %s visibiliy configured %+v", namespace, visibility)
} else if action == deleteEvent {
if val, ok := dm.SystemMonitor.NamespacePidsMap[namespace]; ok {
for _, nskey := range val.NsKeys {
dm.SystemMonitor.UpdateNsKeyMap(deleteEvent, nskey, tp.Visibility{})
}
}
delete(dm.SystemMonitor.NamespacePidsMap, namespace)
}
}
var visibilityKey string = "kubearmor-visibility"
func (dm *KubeArmorDaemon) updateVisibilityWithCM(cm *corev1.ConfigMap, action string) {
dm.SystemMonitor.UpdateVisibility() // update host and global default bpf maps
// get all namespaces
nsList, err := K8s.K8sClient.CoreV1().Namespaces().List(context.Background(), metav1.ListOptions{})
if err != nil {
kg.Err("unable to fetch namespace list")
return
}
// for each namespace if needed change the visibility
for _, ns := range nsList.Items {
// if namespace is annotated with visibility annotation don't update on config map change
if _, found := ns.Annotations[visibilityKey]; found || kl.ContainsElement(dm.SystemMonitor.UntrackedNamespaces, ns.Name) {
continue
}
visibility := tp.Visibility{
File: strings.Contains(cm.Data[cfg.ConfigVisibility], "file"),
Process: strings.Contains(cm.Data[cfg.ConfigVisibility], "process"),
Network: strings.Contains(cm.Data[cfg.ConfigVisibility], "network"),
Capabilities: strings.Contains(cm.Data[cfg.ConfigVisibility], "capabilities"),
DNS: strings.Contains(cm.Data[cfg.ConfigVisibility], "dns"),
}
dm.UpdateVisibility(updateEvent, ns.Name, visibility)
}
}
// UpdateGlobalPosture Function
func (dm *KubeArmorDaemon) UpdateGlobalPosture(posture tp.DefaultPosture) {
dm.EndPointsLock.Lock()
defer dm.EndPointsLock.Unlock()
dm.DefaultPosturesLock.Lock()
defer dm.DefaultPosturesLock.Unlock()
cfg.GlobalCfg.DefaultFilePosture = validateGlobalDefaultPosture(posture.FileAction)
cfg.GlobalCfg.DefaultNetworkPosture = validateGlobalDefaultPosture(posture.NetworkAction)
cfg.GlobalCfg.DefaultCapabilitiesPosture = validateGlobalDefaultPosture(posture.CapabilitiesAction)
dm.Logger.Printf("[Update] Global DefaultPosture {File:%v, Capabilities:%v, Network:%v}",
cfg.GlobalCfg.DefaultFilePosture,
cfg.GlobalCfg.DefaultCapabilitiesPosture,
cfg.GlobalCfg.DefaultNetworkPosture)
}
// WatchDefaultPosture Function
func (dm *KubeArmorDaemon) WatchDefaultPosture() cache.InformerSynced {
factory := informers.NewSharedInformerFactory(K8s.K8sClient, 0)
informer := factory.Core().V1().Namespaces().Informer()
registration, err := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
if ns, ok := obj.(*corev1.Namespace); ok {
fp, fa := validateDefaultPosture("kubearmor-file-posture", ns, cfg.GlobalCfg.DefaultFilePosture)
np, na := validateDefaultPosture("kubearmor-network-posture", ns, cfg.GlobalCfg.DefaultNetworkPosture)
cp, ca := validateDefaultPosture("kubearmor-capabilities-posture", ns, cfg.GlobalCfg.DefaultCapabilitiesPosture)
defaultPosture := tp.DefaultPosture{
FileAction: fp,
NetworkAction: np,
CapabilitiesAction: cp,
}
annotated := fa || na || ca
// Set Visibility to Global Default
visibility := tp.Visibility{
File: dm.validateVisibility("file", cfg.GlobalCfg.Visibility),
Process: dm.validateVisibility("process", cfg.GlobalCfg.Visibility),
Network: dm.validateVisibility("network", cfg.GlobalCfg.Visibility),
Capabilities: dm.validateVisibility("capabilities", cfg.GlobalCfg.Visibility),
DNS: dm.validateVisibility("dns", cfg.GlobalCfg.Visibility),
}
// Set Visibility to Namespace Annotation if exists
if ns.Annotations != nil && ns.Annotations[visibilityKey] != "" {
visibility = tp.Visibility{
File: dm.validateVisibility("file", ns.Annotations[visibilityKey]),
Process: dm.validateVisibility("process", ns.Annotations[visibilityKey]),
Network: dm.validateVisibility("network", ns.Annotations[visibilityKey]),
Capabilities: dm.validateVisibility("capabilities", ns.Annotations[visibilityKey]),
DNS: dm.validateVisibility("dns", ns.Annotations[visibilityKey]),
}
}
dm.UpdateDefaultPosture(addEvent, ns.Name, defaultPosture, annotated)
dm.UpdateVisibility(addEvent, ns.Name, visibility)
}
},
UpdateFunc: func(_, new interface{}) {
if ns, ok := new.(*corev1.Namespace); ok {
fp, fa := validateDefaultPosture("kubearmor-file-posture", ns, cfg.GlobalCfg.DefaultFilePosture)
np, na := validateDefaultPosture("kubearmor-network-posture", ns, cfg.GlobalCfg.DefaultNetworkPosture)
cp, ca := validateDefaultPosture("kubearmor-capabilities-posture", ns, cfg.GlobalCfg.DefaultCapabilitiesPosture)
defaultPosture := tp.DefaultPosture{
FileAction: fp,
NetworkAction: np,
CapabilitiesAction: cp,
}
annotated := fa || na || ca
// Set Visibility to Global Default
visibility := tp.Visibility{
File: dm.validateVisibility("file", cfg.GlobalCfg.Visibility),
Process: dm.validateVisibility("process", cfg.GlobalCfg.Visibility),
Network: dm.validateVisibility("network", cfg.GlobalCfg.Visibility),
Capabilities: dm.validateVisibility("capabilities", cfg.GlobalCfg.Visibility),
DNS: dm.validateVisibility("dns", cfg.GlobalCfg.Visibility),
}
// Set Visibility to Namespace Annotation if exists
if ns.Annotations != nil && ns.Annotations[visibilityKey] != "" {
visibility = tp.Visibility{
File: dm.validateVisibility("file", ns.Annotations[visibilityKey]),
Process: dm.validateVisibility("process", ns.Annotations[visibilityKey]),
Network: dm.validateVisibility("network", ns.Annotations[visibilityKey]),
Capabilities: dm.validateVisibility("capabilities", ns.Annotations[visibilityKey]),
DNS: dm.validateVisibility("dns", ns.Annotations[visibilityKey]),
}
}
dm.UpdateDefaultPosture(updateEvent, ns.Name, defaultPosture, annotated)
dm.UpdateVisibility(updateEvent, ns.Name, visibility)
}
},
DeleteFunc: func(obj interface{}) {
if ns, ok := obj.(*corev1.Namespace); ok {
_, fa := validateDefaultPosture("kubearmor-file-posture", ns, cfg.GlobalCfg.DefaultFilePosture)
_, na := validateDefaultPosture("kubearmor-network-posture", ns, cfg.GlobalCfg.DefaultNetworkPosture)
_, ca := validateDefaultPosture("kubearmor-capabilities-posture", ns, cfg.GlobalCfg.DefaultCapabilitiesPosture)
annotated := fa || na || ca
dm.UpdateDefaultPosture(deleteEvent, ns.Name, tp.DefaultPosture{}, annotated)
dm.UpdateVisibility(deleteEvent, ns.Name, tp.Visibility{})
}
},
})
if err != nil {
dm.Logger.Err("Couldn't start watching Default Posture Annotations and namespace")
return nil
}
go factory.Start(StopChan)
return registration.HasSynced
}
// WatchConfigMap function
func (dm *KubeArmorDaemon) WatchConfigMap() cache.InformerSynced {
configMapLabelOption := informers.WithTweakListOptions(func(opts *metav1.ListOptions) {
opts.LabelSelector = fmt.Sprintf("kubearmor-app=%s", "kubearmor-configmap")
})
factory := informers.NewSharedInformerFactoryWithOptions(K8s.K8sClient, 0, configMapLabelOption)
informer := factory.Core().V1().ConfigMaps().Informer()
cmNS := dm.GetConfigMapNS()
var err error
registration, err := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
if cm, ok := obj.(*corev1.ConfigMap); ok && cm.Namespace == cmNS {
cfg.GlobalCfg.HostVisibility = cm.Data[cfg.ConfigHostVisibility]
cfg.GlobalCfg.Visibility = cm.Data[cfg.ConfigVisibility]
cfg.GlobalCfg.Cluster = cm.Data[cfg.ConfigCluster]
dm.NodeLock.Lock()
dm.Node.ClusterName = cm.Data[cfg.ConfigCluster]
dm.NodeLock.Unlock()
if _, ok := cm.Data[cfg.ConfigDefaultPostureLogs]; ok {
cfg.GlobalCfg.DefaultPostureLogs = (cm.Data[cfg.ConfigDefaultPostureLogs] == "true")
}
globalPosture := tp.DefaultPosture{
FileAction: cm.Data[cfg.ConfigDefaultFilePosture],
NetworkAction: cm.Data[cfg.ConfigDefaultNetworkPosture],
CapabilitiesAction: cm.Data[cfg.ConfigDefaultCapabilitiesPosture],
}
currentGlobalPosture := tp.DefaultPosture{
FileAction: cfg.GlobalCfg.DefaultFilePosture,
NetworkAction: cfg.GlobalCfg.DefaultNetworkPosture,
CapabilitiesAction: cfg.GlobalCfg.DefaultCapabilitiesPosture,
}
if _, ok := cm.Data[cfg.ConfigAlertThrottling]; ok {
cfg.GlobalCfg.AlertThrottling = (cm.Data[cfg.ConfigAlertThrottling] == "true")
}
if _, ok := cm.Data[cfg.ConfigMaxAlertPerSec]; ok {
maxAlertPerSec, err := strconv.ParseInt(cm.Data[cfg.ConfigMaxAlertPerSec], 10, 32)
if err != nil {
dm.Logger.Warnf("Error: %s", err)
}
cfg.GlobalCfg.MaxAlertPerSec = int32(maxAlertPerSec)
}
if _, ok := cm.Data[cfg.ConfigThrottleSec]; ok {
throttleSec, err := strconv.ParseInt(cm.Data[cfg.ConfigThrottleSec], 10, 32)
if err != nil {
dm.Logger.Warnf("Error: %s", err)
}
cfg.GlobalCfg.ThrottleSec = int32(throttleSec)
}
if _, ok := cm.Data[cfg.ConfigArgMatching]; ok {
cfg.GlobalCfg.MatchArgs = (cm.Data[cfg.ConfigArgMatching] == "true")
}
dm.SystemMonitor.UpdateThrottlingConfig()
dm.SystemMonitor.UpdateMatchArgsConfig()
dm.Logger.Printf("Current Global Posture is %v", currentGlobalPosture)
dm.UpdateGlobalPosture(globalPosture)
// update default posture for endpoints
dm.updatEndpointsWithCM(cm, addEvent)
// update visibility for namespaces
dm.updateVisibilityWithCM(cm, addEvent)
}
},
UpdateFunc: func(_, new interface{}) {
if cm, ok := new.(*corev1.ConfigMap); ok && cm.Namespace == cmNS {
cfg.GlobalCfg.HostVisibility = cm.Data[cfg.ConfigHostVisibility]
cfg.GlobalCfg.Visibility = cm.Data[cfg.ConfigVisibility]
cfg.GlobalCfg.Cluster = cm.Data[cfg.ConfigCluster]
dm.Node.ClusterName = cm.Data[cfg.ConfigCluster]
if _, ok := cm.Data[cfg.ConfigDefaultPostureLogs]; ok {
cfg.GlobalCfg.DefaultPostureLogs = (cm.Data[cfg.ConfigDefaultPostureLogs] == "true")
}
globalPosture := tp.DefaultPosture{
FileAction: cm.Data[cfg.ConfigDefaultFilePosture],
NetworkAction: cm.Data[cfg.ConfigDefaultNetworkPosture],
CapabilitiesAction: cm.Data[cfg.ConfigDefaultCapabilitiesPosture],
}
currentGlobalPosture := tp.DefaultPosture{
FileAction: cfg.GlobalCfg.DefaultFilePosture,
NetworkAction: cfg.GlobalCfg.DefaultNetworkPosture,
CapabilitiesAction: cfg.GlobalCfg.DefaultCapabilitiesPosture,
}
dm.Logger.Printf("Current Global Posture is %v", currentGlobalPosture)
dm.UpdateGlobalPosture(globalPosture)
// update default posture for endpoints
dm.updatEndpointsWithCM(cm, updateEvent)
// update visibility for namespaces
dm.updateVisibilityWithCM(cm, updateEvent)
if _, ok := cm.Data[cfg.ConfigAlertThrottling]; ok {
cfg.GlobalCfg.AlertThrottling = (cm.Data[cfg.ConfigAlertThrottling] == "true")
}
maxAlertPerSec, err := strconv.ParseInt(cm.Data[cfg.ConfigMaxAlertPerSec], 10, 32)
if err != nil {
dm.Logger.Warnf("Error: %s", err)
}
cfg.GlobalCfg.MaxAlertPerSec = int32(maxAlertPerSec)
throttleSec, err := strconv.ParseInt(cm.Data[cfg.ConfigThrottleSec], 10, 32)
if err != nil {
dm.Logger.Warnf("Error: %s", err)
}
cfg.GlobalCfg.ThrottleSec = int32(throttleSec)
dm.SystemMonitor.UpdateThrottlingConfig()
}
},
DeleteFunc: func(obj interface{}) {
// nothing to do here
},
})
if err != nil {
dm.Logger.Err("Couldn't start watching Configmap")
return nil
}
go factory.Start(StopChan)
return registration.HasSynced
}
// GetConfigMapNS Returns KubeArmor configmap namespace
func (dm *KubeArmorDaemon) GetConfigMapNS() string {
// get namespace from env
envNamespace := os.Getenv("KUBEARMOR_NAMESPACE")
if envNamespace == "" {
// kubearmor is running as system process,
// return "kubearmor" for testing purpose in dev env
return "kubearmor"
}
return envNamespace
}
// SPDX-License-Identifier: Apache-2.0
// Copyright 2021 Authors of KubeArmor
package core
import (
"context"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"github.com/containerd/nri/pkg/api"
"github.com/containerd/nri/pkg/stub"
"github.com/kubearmor/KubeArmor/KubeArmor/common"
kl "github.com/kubearmor/KubeArmor/KubeArmor/common"
cfg "github.com/kubearmor/KubeArmor/KubeArmor/config"
kg "github.com/kubearmor/KubeArmor/KubeArmor/log"
tp "github.com/kubearmor/KubeArmor/KubeArmor/types"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// NRI Handler
var NRI *NRIHandler
type namespaceKey struct {
PidNS uint32
MntNS uint32
}
// namespaceKeyFromContainer creates a namespaceKey from a container.
func namespaceKeyFromContainer(container tp.Container) namespaceKey {
return namespaceKey{
PidNS: container.PidNS,
MntNS: container.MntNS,
}
}
// NRIHandler connects to an NRI socket and informs on container
// creation/deletion events.
type NRIHandler struct {
// NRI plugin stub
stub stub.Stub
// active containers
containers map[string]tp.Container
dm *KubeArmorDaemon
containersByNamespaces map[namespaceKey]string
handleDeletedContainer func(tp.Container)
handleNewContainer func(tp.Container)
}
// NewNRIHandler creates a new NRIHandler with the given event callbacks.
func (dm *KubeArmorDaemon) NewNRIHandler(
handleDeletedContainer func(tp.Container),
handleNewContainer func(tp.Container),
) *NRIHandler {
nri := &NRIHandler{dm: dm}
opts := []stub.Option{
stub.WithSocketPath(cfg.GlobalCfg.NRISocket),
stub.WithPluginIdx(cfg.GlobalCfg.NRIIndex),
stub.WithOnClose(func() {
kg.Printf("restarting NRI")
nri.Start()
}),
}
stub, err := stub.New(nri, opts...)
if err != nil {
kg.Errf("Failed to create NRI stub: %s", err.Error())
return nil
}
nri.containers = map[string]tp.Container{}
nri.containersByNamespaces = map[namespaceKey]string{}
nri.stub = stub
nri.handleDeletedContainer = handleDeletedContainer
nri.handleNewContainer = handleNewContainer
return nri
}
// Start initiates a configured NRI connection.
func (nh *NRIHandler) Start() {
go func() {
err := nh.stub.Run(context.Background())
if err != nil {
kg.Errf("Failed to connect to NRI: %s", err.Error())
}
}()
}
// Stop closes the NRI connection.
func (nh *NRIHandler) Close() {
nh.stub.Stop()
}
// Synchronize is an NRI callback which is called at the beginning of an NRI
// socket connection to inform on all existing containers.
func (nh *NRIHandler) Synchronize(
_ context.Context,
_ []*api.PodSandbox,
nriContainers []*api.Container,
) ([]*api.ContainerUpdate, error) {
for _, nriContainer := range nriContainers {
container := nh.nriToKubeArmorContainer(nriContainer)
container = nh.mergeContainer(container, false)
// Overlapping namespace IDs between containers should be impossible
// here
namespaceKey := namespaceKeyFromContainer(container)
nh.containersByNamespaces[namespaceKey] = container.ContainerID
nh.handleNewContainer(container)
}
return nil, nil
}
// StartContainer is an NRI callback which is called after a container has
// started.
//
// Unfortunately we can't use the CreateContainer or PostCreateContainer NRI
// callbacks because they are called without a PID value, which is required in
// order to get the PID and mount namespaces of the container. This means that
// there is a short period of time between a container starting and us enforcing
// it.
//
// If StartContainer detects a container namespace ID overlap with a previous
// container (since Linux can reuse namespace IDs), it will override the old
// policy correctly, but any actions runc took to set up this container and
// start it will be logged/enforced as if they were the old container's actions.
// This should be exceedingly rare, but there's no way using just NRI that we
// can entirely avoid this scenario.
func (nh *NRIHandler) StartContainer(
_ context.Context,
_ *api.PodSandbox,
nriContainer *api.Container,
) error {
container := nh.nriToKubeArmorContainer(nriContainer)
container = nh.mergeContainer(container, false)
namespaceKey := namespaceKeyFromContainer(container)
// It's technically possible for a container to crash and a new one to be
// started, all before we receive the StopContainer event. And because Linux
// can reuse namespace IDs, it's possible for the enforcement configuration
// to get confused and messed up, so if namespace IDs ever overlap, we
// assume the previous container using those namespaces has already exited.
if oldContainerID, ok := nh.containersByNamespaces[namespaceKey]; ok {
delete(nh.containers, container.ContainerID)
nh.handleDeletedContainer(nh.containers[oldContainerID])
}
nh.containersByNamespaces[namespaceKey] = container.ContainerID
nh.handleNewContainer(container)
return nil
}
// StopContainer is an NRI callback which is called before a container receives
// the signal to stop.
//
// StopContainer is called synchronously before a termination signal is sent to
// a container, so we can be sure that we stop enforcing before the container
// shuts down, at least in most cases. This means that if a new container reuses
// Linux namespace IDs from a previous container, so long as that previous
// container didn't crash unexpectedly, we can be sure that we won't
// accidentally enforce the new container with the old container's policy.
//
// The tradeoff here is that once a container receives its termination signal,
// KubeArmor is no longer enforcing anything on it while it shuts down.
func (nh *NRIHandler) StopContainer(
_ context.Context,
_ *api.PodSandbox,
nriContainer *api.Container,
) ([]*api.ContainerUpdate, error) {
container := nh.nriToKubeArmorContainer(nriContainer)
container = nh.mergeContainer(container, true)
// Only handle the container deleted event if it wasn't already 'deleted' by
// the StartContainer event (due to a Linux namespace ID collision).
if _, ok := nh.containersByNamespaces[namespaceKeyFromContainer(container)]; ok {
delete(nh.containers, container.ContainerID)
nh.handleDeletedContainer(container)
}
return nil, nil
}
// RemoveContainer is an NRI callback which is called after a container has
// exited.
//
// In case StopContainer isn't called, we hook into RemoveContainer to ensure
// that we stop enforcing a container after it has exited. For example, the NRI
// API doesn't guarantee that StopContainer will be called if a container
// crashed unexpectedly.
func (nh *NRIHandler) RemoveContainer(
_ context.Context,
_ *api.PodSandbox,
nriContainer *api.Container,
) ([]*api.ContainerUpdate, error) {
container := nh.nriToKubeArmorContainer(nriContainer)
container = nh.mergeContainer(container, true)
// Only handle the container deleted event if it wasn't already 'deleted' by
// the StartContainer event (due to a Linux namespace ID collision) or
// StopContainer event.
if _, ok := nh.containersByNamespaces[namespaceKeyFromContainer(container)]; ok {
delete(nh.containers, container.ContainerID)
nh.handleDeletedContainer(container)
}
return nil, nil
}
// mergeContainer updates the container with the container's previously-stored
// namespace IDs, if any, also storing namespaceIDs for future reference.
func (nh *NRIHandler) mergeContainer(container tp.Container, removing bool) tp.Container {
if existing, ok := nh.containers[container.ContainerID]; ok {
if existing.PidNS != 0 {
container.PidNS = existing.PidNS
}
if existing.MntNS != 0 {
container.MntNS = existing.MntNS
}
nh.containers[container.ContainerID] = container
} else if !removing {
nh.containers[container.ContainerID] = container
}
return container
}
// nriToKubeArmorContainer converts an NRI container to a KubeArmor container.
func (nh *NRIHandler) nriToKubeArmorContainer(nriContainer *api.Container) tp.Container {
container := tp.Container{}
container.ContainerID = nriContainer.Id
container.ContainerName = nriContainer.Name
container.NamespaceName = "Unknown"
container.EndPointName = "Unknown"
if _, ok := nriContainer.Labels["io.kubernetes.pod.namespace"]; ok {
container.NamespaceName = nriContainer.Labels["io.kubernetes.pod.namespace"] // Pod namespace
if _, ok := nriContainer.Labels["io.kubernetes.pod.name"]; ok {
container.EndPointName = nriContainer.Labels["io.kubernetes.pod.name"] // Pod name
}
}
var podName string
var podNamespace string
if name, ok := nriContainer.Labels["io.kubernetes.pod.name"]; ok {
podName = name
}
if namespace, ok := nriContainer.Labels["io.kubernetes.pod.namespace"]; ok {
podNamespace = namespace
}
if nh.dm.K8sEnabled {
pod, err := K8s.K8sClient.CoreV1().Pods(podNamespace).Get(context.TODO(), podName, metav1.GetOptions{})
if err != nil {
kg.Warnf("failed to fetch Pod: %w\n", err)
}
if appArmorProfile, ok := pod.Annotations["container.apparmor.security.beta.kubernetes.io/"+nriContainer.Name]; ok {
profile := strings.Split(appArmorProfile, "/")
if len(profile) > 1 {
container.AppArmorProfile = profile[1]
}
}
} else {
container.AppArmorProfile = "kubearmor_" + container.ContainerName
}
// Read PID and mount namespaces from container root PID
if nriContainer.Pid != 0 {
pid := strconv.Itoa(int(nriContainer.Pid))
if data, err := os.Readlink(filepath.Join(cfg.GlobalCfg.ProcFsMount, pid, "/ns/pid")); err == nil {
if _, err := fmt.Sscanf(data, "pid:[%d]", &container.PidNS); err != nil {
kg.Warnf("Unable to get PidNS (%s, %s, %s)", nriContainer.Id, nriContainer.Pid, err.Error())
}
}
if data, err := os.Readlink(filepath.Join(cfg.GlobalCfg.ProcFsMount, pid, "/ns/mnt")); err == nil {
if _, err := fmt.Sscanf(data, "mnt:[%d]", &container.MntNS); err != nil {
kg.Warnf("Unable to get MntNS (%s, %s, %s)", nriContainer.Id, nriContainer.Pid, err.Error())
}
}
}
return container
}
// MonitorNRIEvents monitors NRI events.
func (dm *KubeArmorDaemon) MonitorNRIEvents() {
dm.WgDaemon.Add(1)
defer dm.WgDaemon.Done()
handleNewContainer := func(container tp.Container) {
endpoint := tp.EndPoint{}
dm.ContainersLock.Lock()
if len(dm.OwnerInfo) > 0 {
if podOwnerInfo, ok := dm.OwnerInfo[container.EndPointName]; ok {
container.Owner = podOwnerInfo
}
}
if _, ok := dm.Containers[container.ContainerID]; !ok {
dm.Containers[container.ContainerID] = container
dm.ContainersLock.Unlock()
} else if dm.Containers[container.ContainerID].PidNS == 0 && dm.Containers[container.ContainerID].MntNS == 0 {
// this entry was updated by kubernetes before docker detects it
// thus, we here use the info given by kubernetes instead of the info given by docker
container.NamespaceName = dm.Containers[container.ContainerID].NamespaceName
container.EndPointName = dm.Containers[container.ContainerID].EndPointName
container.Labels = dm.Containers[container.ContainerID].Labels
container.ContainerName = dm.Containers[container.ContainerID].ContainerName
container.ContainerImage = dm.Containers[container.ContainerID].ContainerImage
container.PolicyEnabled = dm.Containers[container.ContainerID].PolicyEnabled
container.ProcessVisibilityEnabled = dm.Containers[container.ContainerID].ProcessVisibilityEnabled
container.FileVisibilityEnabled = dm.Containers[container.ContainerID].FileVisibilityEnabled
container.NetworkVisibilityEnabled = dm.Containers[container.ContainerID].NetworkVisibilityEnabled
container.CapabilitiesVisibilityEnabled = dm.Containers[container.ContainerID].CapabilitiesVisibilityEnabled
dm.Containers[container.ContainerID] = container
dm.ContainersLock.Unlock()
dm.EndPointsLock.Lock()
for idx, endPoint := range dm.EndPoints {
if endPoint.NamespaceName == container.NamespaceName && endPoint.EndPointName == container.EndPointName && kl.ContainsElement(endPoint.Containers, container.ContainerID) {
// update containers
if !kl.ContainsElement(endPoint.Containers, container.ContainerID) { // does not make sense but need to verify
dm.EndPoints[idx].Containers = append(dm.EndPoints[idx].Containers, container.ContainerID)
}
endpoint = dm.EndPoints[idx]
break
}
}
dm.EndPointsLock.Unlock()
} else {
dm.ContainersLock.Unlock()
return
}
if dm.SystemMonitor != nil && cfg.GlobalCfg.Policy {
// for throttling
dm.SystemMonitor.Logger.ContainerNsKey[container.ContainerID] = common.OuterKey{
MntNs: container.MntNS,
PidNs: container.PidNS,
}
// update NsMap
dm.SystemMonitor.AddContainerIDToNsMap(container.ContainerID, container.NamespaceName, container.PidNS, container.MntNS)
dm.RuntimeEnforcer.RegisterContainer(container.ContainerID, container.PidNS, container.MntNS)
if dm.Presets != nil {
dm.Presets.RegisterContainer(container.ContainerID, container.PidNS, container.MntNS)
}
if len(endpoint.SecurityPolicies) > 0 { // struct can be empty or no policies registered for the endpoint yet
dm.Logger.UpdateSecurityPolicies("ADDED", endpoint)
if dm.RuntimeEnforcer != nil && endpoint.PolicyEnabled == tp.KubeArmorPolicyEnabled {
// enforce security policies
dm.RuntimeEnforcer.UpdateSecurityPolicies(endpoint)
}
if dm.Presets != nil && endpoint.PolicyEnabled == tp.KubeArmorPolicyEnabled {
// enforce preset rules
dm.Presets.UpdateSecurityPolicies(endpoint)
}
}
}
if !dm.K8sEnabled {
dm.ContainersLock.Lock()
dm.EndPointsLock.Lock()
dm.MatchandUpdateContainerSecurityPolicies(container.ContainerID)
dm.EndPointsLock.Unlock()
dm.ContainersLock.Unlock()
}
dm.Logger.Printf("Detected a container (added/%.12s/pidns=%d/mntns=%d)", container.ContainerID, container.PidNS, container.MntNS)
}
handleDeletedContainer := func(container tp.Container) {
dm.ContainersLock.Lock()
_, ok := dm.Containers[container.ContainerID]
if !ok {
dm.ContainersLock.Unlock()
return
}
if !dm.K8sEnabled {
dm.EndPointsLock.Lock()
dm.MatchandRemoveContainerFromEndpoint(container.ContainerID)
dm.EndPointsLock.Unlock()
}
delete(dm.Containers, container.ContainerID)
dm.ContainersLock.Unlock()
if dm.SystemMonitor != nil && cfg.GlobalCfg.Policy {
outkey := dm.SystemMonitor.Logger.ContainerNsKey[container.ContainerID]
dm.Logger.DeleteAlertMapKey(outkey)
delete(dm.SystemMonitor.Logger.ContainerNsKey, container.ContainerID)
// update NsMap
dm.SystemMonitor.DeleteContainerIDFromNsMap(container.ContainerID, container.NamespaceName, container.PidNS, container.MntNS)
dm.RuntimeEnforcer.UnregisterContainer(container.ContainerID)
if dm.Presets != nil {
dm.Presets.UnregisterContainer(container.ContainerID)
}
}
dm.Logger.Printf("Detected a container (removed/%.12s/pidns=%d/mntns=%d)", container.ContainerID, container.PidNS, container.MntNS)
}
NRI = dm.NewNRIHandler(handleDeletedContainer, handleNewContainer)
// check if NRI exists
if NRI == nil {
return
}
NRI.Start()
dm.Logger.Print("Started to monitor NRI events")
}
// SPDX-License-Identifier: Apache-2.0
// Copyright 2023 Authors of KubeArmor
package core
import (
"encoding/json"
"os"
"regexp"
"sort"
"strings"
"github.com/fsnotify/fsnotify"
"github.com/spf13/viper"
kl "github.com/kubearmor/KubeArmor/KubeArmor/common"
cfg "github.com/kubearmor/KubeArmor/KubeArmor/config"
kg "github.com/kubearmor/KubeArmor/KubeArmor/log"
tp "github.com/kubearmor/KubeArmor/KubeArmor/types"
pb "github.com/kubearmor/KubeArmor/protobuf"
)
// SetContainerVisibility function enables visibility flag arguments for un-orchestrated container and updates the visibility map
func (dm *KubeArmorDaemon) SetContainerNSVisibility() {
visibility := tp.Visibility{}
if strings.Contains(cfg.GlobalCfg.Visibility, "process") {
visibility.Process = true
}
if strings.Contains(cfg.GlobalCfg.Visibility, "file") {
visibility.File = true
}
if strings.Contains(cfg.GlobalCfg.Visibility, "network") {
visibility.Network = true
}
if strings.Contains(cfg.GlobalCfg.Visibility, "capabilities") {
visibility.Capabilities = true
}
if strings.Contains(cfg.GlobalCfg.Visibility, "dns") {
visibility.DNS = true
}
dm.UpdateVisibility("ADDED", "container_namespace", visibility)
}
// =================== //
// == Config Update == //
// =================== //
// WatchConfigChanges watches for configuration changes and updates the default posture
func (dm *KubeArmorDaemon) WatchConfigChanges() {
viper.OnConfigChange(func(e fsnotify.Event) {
dm.Logger.Printf("Config file changed: %s", e.Name)
cfg.LoadDynamicConfig()
// Update the default posture
globalPosture := tp.DefaultPosture{
FileAction: validateGlobalDefaultPosture(cfg.GlobalCfg.DefaultFilePosture),
NetworkAction: validateGlobalDefaultPosture(cfg.GlobalCfg.DefaultNetworkPosture),
CapabilitiesAction: validateGlobalDefaultPosture(cfg.GlobalCfg.DefaultCapabilitiesPosture),
}
// Update the visibility
visibility := tp.Visibility{
File: dm.validateVisibility("file", cfg.GlobalCfg.Visibility),
Process: dm.validateVisibility("process", cfg.GlobalCfg.Visibility),
Network: dm.validateVisibility("network", cfg.GlobalCfg.Visibility),
Capabilities: dm.validateVisibility("capabilities", cfg.GlobalCfg.Visibility),
DNS: dm.validateVisibility("dns", cfg.GlobalCfg.Visibility),
}
// Apply the changes to the daemon
dm.UpdateGlobalPosture(globalPosture)
// Update default posture for endpoints
for _, ep := range dm.EndPoints {
dm.Logger.Printf("Updating Default Posture for endpoint %s", ep.EndPointName)
dm.UpdateDefaultPosture("MODIFIED", ep.NamespaceName, globalPosture, false)
dm.UpdateVisibility("MODIFIED", ep.NamespaceName, visibility)
}
// Update throttling configs
dm.SystemMonitor.UpdateThrottlingConfig()
// Update the default posture and visibility for the unorchestrated containers
dm.SystemMonitor.UpdateVisibility()
dm.UpdateHostSecurityPolicies()
})
viper.WatchConfig()
}
// ====================================== //
// == Container Security Policy Update == //
// ====================================== //
// MatchandUpdateContainerSecurityPolicies finds relevant endpoint for containers and updates the security policies for enforcement
func (dm *KubeArmorDaemon) MatchandUpdateContainerSecurityPolicies(cid string) {
container := dm.Containers[cid]
for idx, ep := range dm.EndPoints {
_, containerIdentities := kl.GetLabelsFromString(container.Labels)
if ep.EndPointName == dm.Containers[cid].ContainerName || kl.MatchIdentities(ep.Identities, containerIdentities) {
ep.Containers = append(ep.Containers, cid)
dm.EndPoints[idx] = ep
ctr := dm.Containers[cid]
ctr.NamespaceName = ep.NamespaceName
ctr.EndPointName = ep.EndPointName
dm.Containers[cid] = ctr
if cfg.GlobalCfg.Policy {
// update security policies
dm.Logger.UpdateSecurityPolicies("MODIFIED", ep)
if ep.PolicyEnabled == tp.KubeArmorPolicyEnabled {
if dm.RuntimeEnforcer != nil {
// enforce security policies
dm.RuntimeEnforcer.UpdateSecurityPolicies(ep)
}
if dm.Presets != nil {
dm.Presets.UpdateSecurityPolicies(ep)
}
}
}
}
}
}
// MatchandRemoveContainerSecurityPolicies finds relevant endpoint for containers and removes cid from the container list
func (dm *KubeArmorDaemon) MatchandRemoveContainerFromEndpoint(cid string) {
container := dm.Containers[cid]
for idx, ep := range dm.EndPoints {
_, containerIdentities := kl.GetLabelsFromString(container.Labels)
if ep.EndPointName == container.ContainerName || kl.MatchIdentities(ep.Identities, containerIdentities) {
for i, c := range ep.Containers {
if c != cid {
continue
}
ep.Containers = append(ep.Containers[:i], ep.Containers[i+1:]...)
break
}
}
dm.EndPoints[idx] = ep
}
}
func (dm *KubeArmorDaemon) handlePolicyEvent(eventType string, createEndPoint bool, secPolicy tp.SecurityPolicy, newPoint tp.EndPoint, endpointIdx int, containername string) (int, pb.PolicyStatus) {
if containername == "" {
containername = newPoint.ContainerName
}
appArmorAnnotations := map[string]string{}
appArmorAnnotations[containername] = "kubearmor_" + containername
globalDefaultPosture := tp.DefaultPosture{
FileAction: cfg.GlobalCfg.DefaultFilePosture,
NetworkAction: cfg.GlobalCfg.DefaultNetworkPosture,
CapabilitiesAction: cfg.GlobalCfg.DefaultCapabilitiesPosture,
}
newPoint.DefaultPosture = globalDefaultPosture
// check that a security policy should exist before performing delete operation
policymatch := 0
for _, policy := range newPoint.SecurityPolicies {
// check if policy exist
if policy.Metadata["namespaceName"] == secPolicy.Metadata["namespaceName"] && policy.Metadata["policyName"] == secPolicy.Metadata["policyName"] {
policymatch = 1 // policy exists
}
}
// policy doesn't exist and the policy is being removed
if policymatch == 0 && eventType == "DELETED" {
dm.Logger.Warnf("Failed to delete security policy. Policy doesn't exist")
return endpointIdx, pb.PolicyStatus_NotExist
}
for idx, policy := range newPoint.SecurityPolicies {
if policy.Metadata["namespaceName"] == secPolicy.Metadata["namespaceName"] && policy.Metadata["policyName"] == secPolicy.Metadata["policyName"] {
if eventType == "DELETED" {
newPoint.SecurityPolicies = append(newPoint.SecurityPolicies[:idx], newPoint.SecurityPolicies[idx+1:]...)
break
} else {
// Policy already exists so modify
eventType = "MODIFIED"
newPoint.SecurityPolicies[idx] = secPolicy
}
}
}
var privilegedProfiles map[string]struct{}
if eventType == "ADDED" {
dm.RuntimeEnforcer.UpdateAppArmorProfiles(containername, "ADDED", appArmorAnnotations, privilegedProfiles)
newPoint.SecurityPolicies = append(newPoint.SecurityPolicies, secPolicy)
if createEndPoint {
// Create new EndPoint - possible scenarios:
// policy received before container
newPoint.NamespaceName = secPolicy.Metadata["namespaceName"]
newPoint.EndPointName = containername
newPoint.ContainerName = containername
newPoint.PolicyEnabled = tp.KubeArmorPolicyEnabled
newPoint.Identities = secPolicy.Spec.Selector.Identities
newPoint.ProcessVisibilityEnabled = true
newPoint.FileVisibilityEnabled = true
newPoint.NetworkVisibilityEnabled = true
newPoint.CapabilitiesVisibilityEnabled = true
newPoint.Containers = []string{}
newPoint.PrivilegedContainers = map[string]struct{}{}
newPoint.AppArmorProfiles = []string{"kubearmor_" + containername}
// add the endpoint into the endpoint list
dm.EndPoints = append(dm.EndPoints, newPoint)
} else {
dm.EndPoints[endpointIdx] = newPoint
}
if cfg.GlobalCfg.Policy {
// update security policies
dm.Logger.UpdateSecurityPolicies("ADDED", newPoint)
if newPoint.PolicyEnabled == tp.KubeArmorPolicyEnabled {
if dm.RuntimeEnforcer != nil {
// enforce security policies
dm.RuntimeEnforcer.UpdateSecurityPolicies(newPoint)
}
if dm.Presets != nil {
dm.Presets.UpdateSecurityPolicies(newPoint)
}
}
}
} else if eventType == "MODIFIED" {
dm.EndPoints[endpointIdx] = newPoint
if cfg.GlobalCfg.Policy {
// update security policies
dm.Logger.UpdateSecurityPolicies("MODIFIED", newPoint)
if newPoint.PolicyEnabled == tp.KubeArmorPolicyEnabled {
if dm.RuntimeEnforcer != nil {
// enforce security policies
dm.RuntimeEnforcer.UpdateSecurityPolicies(newPoint)
}
if dm.Presets != nil {
// enforce preset rules
dm.Presets.UpdateSecurityPolicies(newPoint)
}
}
}
} else { // DELETED
// update security policies after policy deletion
if endpointIdx >= 0 {
dm.EndPoints[endpointIdx] = newPoint
dm.Logger.UpdateSecurityPolicies("DELETED", newPoint)
dm.RuntimeEnforcer.UpdateSecurityPolicies(newPoint)
if dm.Presets != nil {
dm.Presets.UpdateSecurityPolicies(newPoint)
}
// delete endpoint if no containers or policies
if len(newPoint.Containers) == 0 && len(newPoint.SecurityPolicies) == 0 {
dm.EndPoints = append(dm.EndPoints[:endpointIdx], dm.EndPoints[endpointIdx+1:]...)
// since the length of endpoints slice reduced
endpointIdx--
}
}
}
return endpointIdx, pb.PolicyStatus_Applied
}
// ParseAndUpdateContainerSecurityPolicy Function
func (dm *KubeArmorDaemon) ParseAndUpdateContainerSecurityPolicy(event tp.K8sKubeArmorPolicyEvent) pb.PolicyStatus {
// create a container security policy
secPolicy := tp.SecurityPolicy{}
secPolicy.Metadata = map[string]string{}
secPolicy.Metadata["namespaceName"] = "container_namespace" //event.Object.Metadata.Namespace
secPolicy.Metadata["policyName"] = event.Object.Metadata.Name
if err := kl.Clone(event.Object.Spec, &secPolicy.Spec); err != nil {
dm.Logger.Errf("Failed to clone a spec (%s)", err.Error())
return pb.PolicyStatus_Failure
}
kl.ObjCommaExpandFirstDupOthers(&secPolicy.Spec.Network.MatchProtocols)
kl.ObjCommaExpandFirstDupOthers(&secPolicy.Spec.Capabilities.MatchCapabilities)
switch secPolicy.Spec.Action {
case "allow":
secPolicy.Spec.Action = "Allow"
case "audit":
secPolicy.Spec.Action = "Audit"
case "block":
secPolicy.Spec.Action = "Block"
case "":
secPolicy.Spec.Action = "Block" // by default
}
// add identities
if len(secPolicy.Spec.Selector.MatchLabels) == 0 {
dm.Logger.Warnf("Failed to apply policy. No labels to match found on policy.")
return pb.PolicyStatus_Invalid
}
// can't use the container name label and label selectors at the same time
if _, ok := secPolicy.Spec.Selector.MatchLabels["kubearmor.io/container.name"]; ok && len(secPolicy.Spec.Selector.MatchLabels) > 1 {
dm.Logger.Warnf("Failed to apply policy. Cannot use \"kubearmor.io/container.name\" and other labels together.")
return pb.PolicyStatus_Invalid
} else if !ok && dm.RuntimeEnforcer != nil && dm.RuntimeEnforcer.EnforcerType == "AppArmor" {
// this label is necessary in apparmor because profile needs to be created before container
dm.Logger.Warnf("Received policy for AppArmor enforcer without \"kubearmor.io/container.name\"")
return pb.PolicyStatus_Invalid
}
secPolicy.Spec.Selector.Identities = []string{"namespaceName=" + secPolicy.Metadata["namespaceName"]}
containername := ""
for k, v := range secPolicy.Spec.Selector.MatchLabels {
secPolicy.Spec.Selector.Identities = append(secPolicy.Spec.Selector.Identities, k+"="+v)
// TODO: regex based matching
if k == "kubearmor.io/container.name" {
expr, err := regexp.CompilePOSIX(v)
if err != nil {
dm.Logger.Warnf("Failed to parse expression for \"kubearmor.io/container.name\": %s", err.Error())
return pb.PolicyStatus_Invalid
}
containername = expr.String()
}
}
sort.Slice(secPolicy.Spec.Selector.Identities, func(i, j int) bool {
return secPolicy.Spec.Selector.Identities[i] < secPolicy.Spec.Selector.Identities[j]
})
// add severities, tags, messages, and actions
if len(secPolicy.Spec.Process.MatchPaths) > 0 {
for idx, path := range secPolicy.Spec.Process.MatchPaths {
if path.Severity == 0 {
if secPolicy.Spec.Process.Severity != 0 {
secPolicy.Spec.Process.MatchPaths[idx].Severity = secPolicy.Spec.Process.Severity
} else {
secPolicy.Spec.Process.MatchPaths[idx].Severity = secPolicy.Spec.Severity
}
}
if len(path.Tags) == 0 {
if len(secPolicy.Spec.Process.Tags) > 0 {
secPolicy.Spec.Process.MatchPaths[idx].Tags = secPolicy.Spec.Process.Tags
} else {
secPolicy.Spec.Process.MatchPaths[idx].Tags = secPolicy.Spec.Tags
}
}
if len(path.Message) == 0 {
if len(secPolicy.Spec.Process.Message) > 0 {
secPolicy.Spec.Process.MatchPaths[idx].Message = secPolicy.Spec.Process.Message
} else {
secPolicy.Spec.Process.MatchPaths[idx].Message = secPolicy.Spec.Message
}
}
if len(path.Action) == 0 {
if len(secPolicy.Spec.Process.Action) > 0 {
secPolicy.Spec.Process.MatchPaths[idx].Action = secPolicy.Spec.Process.Action
} else {
secPolicy.Spec.Process.MatchPaths[idx].Action = secPolicy.Spec.Action
}
}
}
}
if len(secPolicy.Spec.Process.MatchDirectories) > 0 {
for idx, dir := range secPolicy.Spec.Process.MatchDirectories {
if dir.Severity == 0 {
if secPolicy.Spec.Process.Severity != 0 {
secPolicy.Spec.Process.MatchDirectories[idx].Severity = secPolicy.Spec.Process.Severity
} else {
secPolicy.Spec.Process.MatchDirectories[idx].Severity = secPolicy.Spec.Severity
}
}
if len(dir.Tags) == 0 {
if len(secPolicy.Spec.Process.Tags) > 0 {
secPolicy.Spec.Process.MatchDirectories[idx].Tags = secPolicy.Spec.Process.Tags
} else {
secPolicy.Spec.Process.MatchDirectories[idx].Tags = secPolicy.Spec.Tags
}
}
if len(dir.Message) == 0 {
if len(secPolicy.Spec.Process.Message) > 0 {
secPolicy.Spec.Process.MatchDirectories[idx].Message = secPolicy.Spec.Process.Message
} else {
secPolicy.Spec.Process.MatchDirectories[idx].Message = secPolicy.Spec.Message
}
}
if len(dir.Action) == 0 {
if len(secPolicy.Spec.Process.Action) > 0 {
secPolicy.Spec.Process.MatchDirectories[idx].Action = secPolicy.Spec.Process.Action
} else {
secPolicy.Spec.Process.MatchDirectories[idx].Action = secPolicy.Spec.Action
}
}
}
}
if len(secPolicy.Spec.Process.MatchPatterns) > 0 {
for idx, pat := range secPolicy.Spec.Process.MatchPatterns {
if pat.Severity == 0 {
if secPolicy.Spec.Process.Severity != 0 {
secPolicy.Spec.Process.MatchPatterns[idx].Severity = secPolicy.Spec.Process.Severity
} else {
secPolicy.Spec.Process.MatchPatterns[idx].Severity = secPolicy.Spec.Severity
}
}
if len(pat.Tags) == 0 {
if len(secPolicy.Spec.Process.Tags) > 0 {
secPolicy.Spec.Process.MatchPatterns[idx].Tags = secPolicy.Spec.Process.Tags
} else {
secPolicy.Spec.Process.MatchPatterns[idx].Tags = secPolicy.Spec.Tags
}
}
if len(pat.Message) == 0 {
if len(secPolicy.Spec.Process.Message) > 0 {
secPolicy.Spec.Process.MatchPatterns[idx].Message = secPolicy.Spec.Process.Message
} else {
secPolicy.Spec.Process.MatchPatterns[idx].Message = secPolicy.Spec.Message
}
}
if len(pat.Action) == 0 {
if len(secPolicy.Spec.Process.Action) > 0 {
secPolicy.Spec.Process.MatchPatterns[idx].Action = secPolicy.Spec.Process.Action
} else {
secPolicy.Spec.Process.MatchPatterns[idx].Action = secPolicy.Spec.Action
}
}
}
}
if len(secPolicy.Spec.File.MatchPaths) > 0 {
for idx, path := range secPolicy.Spec.File.MatchPaths {
if path.Severity == 0 {
if secPolicy.Spec.File.Severity != 0 {
secPolicy.Spec.File.MatchPaths[idx].Severity = secPolicy.Spec.File.Severity
} else {
secPolicy.Spec.File.MatchPaths[idx].Severity = secPolicy.Spec.Severity
}
}
if len(path.Tags) == 0 {
if len(secPolicy.Spec.File.Tags) > 0 {
secPolicy.Spec.File.MatchPaths[idx].Tags = secPolicy.Spec.File.Tags
} else {
secPolicy.Spec.File.MatchPaths[idx].Tags = secPolicy.Spec.Tags
}
}
if len(path.Message) == 0 {
if len(secPolicy.Spec.File.Message) > 0 {
secPolicy.Spec.File.MatchPaths[idx].Message = secPolicy.Spec.File.Message
} else {
secPolicy.Spec.File.MatchPaths[idx].Message = secPolicy.Spec.Message
}
}
if len(path.Action) == 0 {
if len(secPolicy.Spec.File.Action) > 0 {
secPolicy.Spec.File.MatchPaths[idx].Action = secPolicy.Spec.File.Action
} else {
secPolicy.Spec.File.MatchPaths[idx].Action = secPolicy.Spec.Action
}
}
}
}
if len(secPolicy.Spec.File.MatchDirectories) > 0 {
for idx, dir := range secPolicy.Spec.File.MatchDirectories {
if dir.Severity == 0 {
if secPolicy.Spec.File.Severity != 0 {
secPolicy.Spec.File.MatchDirectories[idx].Severity = secPolicy.Spec.File.Severity
} else {
secPolicy.Spec.File.MatchDirectories[idx].Severity = secPolicy.Spec.Severity
}
}
if len(dir.Tags) == 0 {
if len(secPolicy.Spec.File.Tags) > 0 {
secPolicy.Spec.File.MatchDirectories[idx].Tags = secPolicy.Spec.File.Tags
} else {
secPolicy.Spec.File.MatchDirectories[idx].Tags = secPolicy.Spec.Tags
}
}
if len(dir.Message) == 0 {
if len(secPolicy.Spec.File.Message) > 0 {
secPolicy.Spec.File.MatchDirectories[idx].Message = secPolicy.Spec.File.Message
} else {
secPolicy.Spec.File.MatchDirectories[idx].Message = secPolicy.Spec.Message
}
}
if len(dir.Action) == 0 {
if len(secPolicy.Spec.File.Action) > 0 {
secPolicy.Spec.File.MatchDirectories[idx].Action = secPolicy.Spec.File.Action
} else {
secPolicy.Spec.File.MatchDirectories[idx].Action = secPolicy.Spec.Action
}
}
}
}
if len(secPolicy.Spec.File.MatchPatterns) > 0 {
for idx, pat := range secPolicy.Spec.File.MatchPatterns {
if pat.Severity == 0 {
if secPolicy.Spec.File.Severity != 0 {
secPolicy.Spec.File.MatchPatterns[idx].Severity = secPolicy.Spec.File.Severity
} else {
secPolicy.Spec.File.MatchPatterns[idx].Severity = secPolicy.Spec.Severity
}
}
if len(pat.Tags) == 0 {
if len(secPolicy.Spec.File.Tags) > 0 {
secPolicy.Spec.File.MatchPatterns[idx].Tags = secPolicy.Spec.File.Tags
} else {
secPolicy.Spec.File.MatchPatterns[idx].Tags = secPolicy.Spec.Tags
}
}
if len(pat.Message) == 0 {
if len(secPolicy.Spec.File.Message) > 0 {
secPolicy.Spec.File.MatchPatterns[idx].Message = secPolicy.Spec.File.Message
} else {
secPolicy.Spec.File.MatchPatterns[idx].Message = secPolicy.Spec.Message
}
}
if len(pat.Action) == 0 {
if len(secPolicy.Spec.File.Action) > 0 {
secPolicy.Spec.File.MatchPatterns[idx].Action = secPolicy.Spec.File.Action
} else {
secPolicy.Spec.File.MatchPatterns[idx].Action = secPolicy.Spec.Action
}
}
}
}
if len(secPolicy.Spec.Network.MatchProtocols) > 0 {
for idx, proto := range secPolicy.Spec.Network.MatchProtocols {
if proto.Severity == 0 {
if secPolicy.Spec.Network.Severity != 0 {
secPolicy.Spec.Network.MatchProtocols[idx].Severity = secPolicy.Spec.Network.Severity
} else {
secPolicy.Spec.Network.MatchProtocols[idx].Severity = secPolicy.Spec.Severity
}
}
if len(proto.Tags) == 0 {
if len(secPolicy.Spec.Network.Tags) > 0 {
secPolicy.Spec.Network.MatchProtocols[idx].Tags = secPolicy.Spec.Network.Tags
} else {
secPolicy.Spec.Network.MatchProtocols[idx].Tags = secPolicy.Spec.Tags
}
}
if len(proto.Message) == 0 {
if len(secPolicy.Spec.Network.Message) > 0 {
secPolicy.Spec.Network.MatchProtocols[idx].Message = secPolicy.Spec.Network.Message
} else {
secPolicy.Spec.Network.MatchProtocols[idx].Message = secPolicy.Spec.Message
}
}
if len(proto.Action) == 0 {
if len(secPolicy.Spec.Network.Action) > 0 {
secPolicy.Spec.Network.MatchProtocols[idx].Action = secPolicy.Spec.Network.Action
} else {
secPolicy.Spec.Network.MatchProtocols[idx].Action = secPolicy.Spec.Action
}
}
}
}
if len(secPolicy.Spec.Capabilities.MatchCapabilities) > 0 {
for idx, cap := range secPolicy.Spec.Capabilities.MatchCapabilities {
if cap.Severity == 0 {
if secPolicy.Spec.Capabilities.Severity != 0 {
secPolicy.Spec.Capabilities.MatchCapabilities[idx].Severity = secPolicy.Spec.Capabilities.Severity
} else {
secPolicy.Spec.Capabilities.MatchCapabilities[idx].Severity = secPolicy.Spec.Severity
}
}
if len(cap.Tags) == 0 {
if len(secPolicy.Spec.Capabilities.Tags) > 0 {
secPolicy.Spec.Capabilities.MatchCapabilities[idx].Tags = secPolicy.Spec.Capabilities.Tags
} else {
secPolicy.Spec.Capabilities.MatchCapabilities[idx].Tags = secPolicy.Spec.Tags
}
}
if len(cap.Message) == 0 {
if len(secPolicy.Spec.Capabilities.Message) > 0 {
secPolicy.Spec.Capabilities.MatchCapabilities[idx].Message = secPolicy.Spec.Capabilities.Message
} else {
secPolicy.Spec.Capabilities.MatchCapabilities[idx].Message = secPolicy.Spec.Message
}
}
if len(cap.Action) == 0 {
if len(secPolicy.Spec.Capabilities.Action) > 0 {
secPolicy.Spec.Capabilities.MatchCapabilities[idx].Action = secPolicy.Spec.Capabilities.Action
} else {
secPolicy.Spec.Capabilities.MatchCapabilities[idx].Action = secPolicy.Spec.Action
}
}
}
}
// handle updates to global policy store
if event.Type == "ADDED" {
dm.SecurityPoliciesLock.Lock()
newPolicy := true
for idx, policy := range dm.SecurityPolicies {
if policy.Metadata["namespaceName"] == secPolicy.Metadata["namespaceName"] && policy.Metadata["policyName"] == secPolicy.Metadata["policyName"] {
// update
newPolicy = false
dm.SecurityPolicies[idx] = secPolicy
break
}
}
if newPolicy {
dm.SecurityPolicies = append(dm.SecurityPolicies, secPolicy)
}
dm.SecurityPoliciesLock.Unlock()
} else if event.Type == "DELETED" {
dm.SecurityPoliciesLock.Lock()
for idx, policy := range dm.SecurityPolicies {
if policy.Metadata["namespaceName"] == secPolicy.Metadata["namespaceName"] && policy.Metadata["policyName"] == secPolicy.Metadata["policyName"] {
dm.SecurityPolicies = append(dm.SecurityPolicies[:idx], dm.SecurityPolicies[idx+1:]...)
break
}
}
dm.SecurityPoliciesLock.Unlock()
}
dm.Logger.Printf("Detected a Container Security Policy (%s/%s/%s)", strings.ToLower(event.Type), secPolicy.Metadata["namespaceName"], secPolicy.Metadata["policyName"])
createEndPoint := true
endPointIndex := -1
newPoint := tp.EndPoint{}
policyStatus := pb.PolicyStatus_Applied
// consider reducing coverage for this lock
dm.EndPointsLock.Lock()
defer dm.EndPointsLock.Unlock()
for idx, endPoint := range dm.EndPoints {
endPointIndex++
// update container rules if there exists another endpoint with same policy.Metadata["policyName"]
// this is for handling cases when an existing policy has been sent with modified identites - we delete security policies
// from previously matched endpoint
for policyIndex, policy := range endPoint.SecurityPolicies {
if policy.Metadata["namespaceName"] == secPolicy.Metadata["namespaceName"] && policy.Metadata["policyName"] == secPolicy.Metadata["policyName"] && !kl.MatchIdentities(secPolicy.Spec.Selector.Identities, endPoint.Identities) {
// if no containers and only policy with this name exists, delete endpoint
if len(endPoint.Containers) == 0 && len(endPoint.SecurityPolicies) == 1 {
dm.EndPoints = append(dm.EndPoints[:idx], dm.EndPoints[idx+1:]...)
// delete unnecessary security policies
dm.Logger.UpdateSecurityPolicies("DELETED", endPoint)
endPoint.SecurityPolicies = append(endPoint.SecurityPolicies[:0], endPoint.SecurityPolicies[1:]...)
dm.RuntimeEnforcer.UpdateSecurityPolicies(endPoint)
if dm.Presets != nil {
dm.Presets.UpdateSecurityPolicies(endPoint)
}
endPoint = tp.EndPoint{}
endPointIndex--
} else if len(endPoint.SecurityPolicies) >= 1 {
// else update the security policies for this endpoint
// as it has multiple containers/policies
dm.EndPoints[idx].SecurityPolicies = append(
dm.EndPoints[idx].SecurityPolicies[:policyIndex],
dm.EndPoints[idx].SecurityPolicies[policyIndex+1:]...,
)
endPoint = dm.EndPoints[idx]
if cfg.GlobalCfg.Policy {
// update security policies
dm.Logger.UpdateSecurityPolicies("MODIFIED", endPoint)
if endPoint.PolicyEnabled == tp.KubeArmorPolicyEnabled {
if dm.RuntimeEnforcer != nil {
// enforce security policies
dm.RuntimeEnforcer.UpdateSecurityPolicies(endPoint)
}
if dm.Presets != nil {
// enforce preset rules
dm.Presets.UpdateSecurityPolicies(endPoint)
}
}
}
}
break
}
}
// update policy for all endpoints that match
if kl.MatchIdentities(secPolicy.Spec.Selector.Identities, endPoint.Identities) {
// endpoint exists for this sec policy, so we update it
createEndPoint = false
newPoint = endPoint
endPointIndex, policyStatus = dm.handlePolicyEvent(event.Type, createEndPoint, secPolicy, newPoint, endPointIndex, containername)
switch policyStatus {
case pb.PolicyStatus_Applied, pb.PolicyStatus_Deleted, pb.PolicyStatus_Modified:
continue
default:
return policyStatus
}
}
}
// endpoint doesn't exist for this policy yet
if createEndPoint {
_, policyStatus = dm.handlePolicyEvent(event.Type, true, secPolicy, newPoint, endPointIndex, containername)
switch policyStatus {
case pb.PolicyStatus_Applied, pb.PolicyStatus_Deleted, pb.PolicyStatus_Modified:
default:
return policyStatus
}
}
// backup/remove container policies
if !dm.K8sEnabled && (cfg.GlobalCfg.KVMAgent || cfg.GlobalCfg.Policy) {
if event.Type == "ADDED" || event.Type == "MODIFIED" {
// backup SecurityPolicy to file
dm.backupKubeArmorContainerPolicy(secPolicy)
} else if event.Type == "DELETED" {
dm.removeBackUpPolicy(secPolicy.Metadata["policyName"])
}
}
if event.Type == "ADDED" {
return pb.PolicyStatus_Applied
} else if event.Type == "DELETED" {
return pb.PolicyStatus_Deleted
}
return pb.PolicyStatus_Modified
}
// ================================= //
// == HostPolicy Backup & Restore == //
// ================================= //
// backupKubeArmorHostPolicy Function
func (dm *KubeArmorDaemon) backupKubeArmorHostPolicy(policy tp.HostSecurityPolicy) {
// Check for "/opt/kubearmor/policies" path. If dir not found, create the same
if _, err := os.Stat(cfg.PolicyDir); err != nil {
if err = os.MkdirAll(cfg.PolicyDir, 0700); err != nil {
kg.Warnf("Dir creation failed for [%v]", cfg.PolicyDir)
return
}
}
var file *os.File
var err error
if file, err = os.Create(cfg.PolicyDir + policy.Metadata["policyName"] + ".yaml"); err == nil {
if policyBytes, err := json.Marshal(policy); err == nil {
if _, err = file.Write(policyBytes); err == nil {
if err := file.Close(); err != nil {
dm.Logger.Err(err.Error())
}
}
}
}
}
// Back up KubeArmor container policies in /opt/kubearmor/policies
func (dm *KubeArmorDaemon) backupKubeArmorContainerPolicy(policy tp.SecurityPolicy) {
// Check for "/opt/kubearmor/policies" path. If dir not found, create the same
if _, err := os.Stat(cfg.PolicyDir); err != nil {
if err = os.MkdirAll(cfg.PolicyDir, 0700); err != nil {
kg.Warnf("Dir creation failed for [%v]", cfg.PolicyDir)
return
}
}
var file *os.File
var err error
if file, err = os.Create(cfg.PolicyDir + policy.Metadata["policyName"] + ".yaml"); err == nil {
if policyBytes, err := json.Marshal(policy); err == nil {
if _, err = file.Write(policyBytes); err == nil {
if err := file.Close(); err != nil {
dm.Logger.Err(err.Error())
}
}
}
}
}
func (dm *KubeArmorDaemon) restoreKubeArmorPolicies() {
if _, err := os.Stat(cfg.PolicyDir); err != nil {
kg.Warn("Policies dir not found for restoration")
return
}
// List all policies files from "/opt/kubearmor/policies" path
if policyFiles, err := os.ReadDir(cfg.PolicyDir); err == nil {
for _, file := range policyFiles {
if data, err := os.ReadFile(cfg.PolicyDir + file.Name()); err == nil {
var k struct {
Metadata map[string]string `json:"metadata"`
}
err := json.Unmarshal(data, &k)
if err != nil {
kg.Errf("Failed to unmarshal policy: %v", err)
continue
}
if _, ok := k.Metadata["namespaceName"]; ok { // ContainerPolicy contains namespaceName
var containerPolicy tp.K8sKubeArmorPolicy
if err := json.Unmarshal(data, &containerPolicy); err == nil {
containerPolicy.Metadata.Name = k.Metadata["policyName"]
dm.ParseAndUpdateContainerSecurityPolicy(tp.K8sKubeArmorPolicyEvent{
Type: "ADDED",
Object: containerPolicy,
})
}
} else { // HostSecurityPolicy
var hostPolicy tp.K8sKubeArmorHostPolicy
if err := json.Unmarshal(data, &hostPolicy); err == nil {
hostPolicy.Metadata.Name = k.Metadata["policyName"]
dm.ParseAndUpdateHostSecurityPolicy(tp.K8sKubeArmorHostPolicyEvent{
Type: "ADDED",
Object: hostPolicy,
})
} else {
kg.Errf("Failed to unmarshal host policy: %v", err)
}
}
}
}
if len(policyFiles) == 0 {
kg.Warn("No policies found for restoration")
}
}
}
// removeBackUpPolicy Function
func (dm *KubeArmorDaemon) removeBackUpPolicy(name string) {
fname := cfg.PolicyDir + name + ".yaml"
// Check for "/opt/kubearmor/policies" path. If dir not found, create the same
if _, err := os.Stat(fname); err != nil {
kg.Printf("Backup policy [%v] not exist", fname)
return
}
if err := os.Remove(fname); err != nil {
kg.Errf("unable to delete file:%s err=%s", fname, err.Error())
}
}