1// +build functional 2 3package cri_containerd 4 5import ( 6 "context" 7 "errors" 8 "flag" 9 "fmt" 10 "net" 11 "os" 12 "testing" 13 "time" 14 15 "github.com/Microsoft/hcsshim/osversion" 16 _ "github.com/Microsoft/hcsshim/test/functional/manifest" 17 testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" 18 "github.com/containerd/containerd" 19 eventtypes "github.com/containerd/containerd/api/events" 20 eventsapi "github.com/containerd/containerd/api/services/events/v1" 21 eventruntime "github.com/containerd/containerd/runtime" 22 "github.com/containerd/typeurl" 23 "github.com/gogo/protobuf/types" 24 "google.golang.org/grpc" 25 runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" 26) 27 28const ( 29 daemonAddress = "tcp://127.0.0.1:2376" 30 connectTimeout = time.Second * 10 31 testNamespace = "cri-containerd-test" 32 33 wcowProcessRuntimeHandler = "runhcs-wcow-process" 34 wcowHypervisorRuntimeHandler = "runhcs-wcow-hypervisor" 35 wcowHypervisor17763RuntimeHandler = "runhcs-wcow-hypervisor-17763" 36 wcowHypervisor18362RuntimeHandler = "runhcs-wcow-hypervisor-18362" 37 wcowHypervisor19041RuntimeHandler = "runhcs-wcow-hypervisor-19041" 38 39 testDeviceUtilFilePath = "C:\\ContainerPlat\\device-util.exe" 40 testDriversPath = "C:\\ContainerPlat\\testdrivers" 41 testGPUBootFiles = "C:\\ContainerPlat\\LinuxBootFiles\\nvidiagpu" 42 43 lcowRuntimeHandler = "runhcs-lcow" 44 imageLcowK8sPause = "k8s.gcr.io/pause:3.1" 45 imageLcowAlpine = "docker.io/library/alpine:latest" 46 imageLcowCosmos = "cosmosarno/spark-master:2.4.1_2019-04-18_8e864ce" 47 alpineAspNet = "mcr.microsoft.com/dotnet/core/aspnet:3.1-alpine3.11" 48 alpineAspnetUpgrade = "mcr.microsoft.com/dotnet/core/aspnet:3.1.2-alpine3.11" 49 // Default account name for use with GMSA related tests. This will not be 50 // present/you will not have access to the account on your machine unless 51 // your environment is configured properly. 52 gmsaAccount = "cplat" 53) 54 55// Image definitions 56var ( 57 imageWindowsNanoserver = getWindowsNanoserverImage(osversion.Get().Build) 58 imageWindowsServercore = getWindowsServerCoreImage(osversion.Get().Build) 59 imageWindowsNanoserver17763 = getWindowsNanoserverImage(osversion.RS5) 60 imageWindowsNanoserver18362 = getWindowsNanoserverImage(osversion.V19H1) 61 imageWindowsNanoserver19041 = getWindowsNanoserverImage(osversion.V20H1) 62 imageWindowsServercore17763 = getWindowsServerCoreImage(osversion.RS5) 63 imageWindowsServercore18362 = getWindowsServerCoreImage(osversion.V19H1) 64 imageWindowsServercore19041 = getWindowsServerCoreImage(osversion.V20H1) 65) 66 67// Flags 68var ( 69 flagFeatures = testutilities.NewStringSetFlag() 70) 71 72// Features 73// Make sure you update allFeatures below with any new features you add. 74const ( 75 featureLCOW = "LCOW" 76 featureWCOWProcess = "WCOWProcess" 77 featureWCOWHypervisor = "WCOWHypervisor" 78 featureGMSA = "GMSA" 79 featureGPU = "GPU" 80) 81 82var allFeatures = []string{ 83 featureLCOW, 84 featureWCOWProcess, 85 featureWCOWHypervisor, 86 featureGMSA, 87 featureGPU, 88} 89 90func init() { 91 // Flag definitions must be in init rather than TestMain, as TestMain isn't 92 // called if -help is passed, but we want the feature usage to show up. 93 flag.Var(flagFeatures, "feature", fmt.Sprintf( 94 "specifies which sets of functionality to test. can be set multiple times\n"+ 95 "supported features: %v", allFeatures)) 96} 97 98func TestMain(m *testing.M) { 99 flag.Parse() 100 os.Exit(m.Run()) 101} 102 103// requireFeatures checks in flagFeatures to validate that each required feature 104// was enabled, and skips the test if any are missing. If the flagFeatures set 105// is empty, the function returns (by default all features are enabled). 106func requireFeatures(t *testing.T, features ...string) { 107 set := flagFeatures.ValueSet() 108 if len(set) == 0 { 109 return 110 } 111 for _, feature := range features { 112 if _, ok := set[feature]; !ok { 113 t.Skipf("skipping test due to feature not set: %s", feature) 114 } 115 } 116} 117 118func getWindowsNanoserverImage(build uint16) string { 119 switch build { 120 case osversion.RS5: 121 return "mcr.microsoft.com/windows/nanoserver:1809" 122 case osversion.V19H1: 123 return "mcr.microsoft.com/windows/nanoserver:1903" 124 case osversion.V20H1: 125 return "mcr.microsoft.com/windows/nanoserver:2004" 126 default: 127 panic("unsupported build") 128 } 129} 130 131func getWindowsServerCoreImage(build uint16) string { 132 switch build { 133 case osversion.RS5: 134 return "mcr.microsoft.com/windows/servercore:1809" 135 case osversion.V19H1: 136 return "mcr.microsoft.com/windows/servercore:1903" 137 case osversion.V20H1: 138 return "mcr.microsoft.com/windows/servercore:2004" 139 default: 140 panic("unsupported build") 141 } 142} 143 144func createGRPCConn(ctx context.Context) (*grpc.ClientConn, error) { 145 conn, err := grpc.DialContext(ctx, daemonAddress, grpc.WithInsecure(), grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) { 146 return net.DialTimeout("tcp", "127.0.0.1:2376", timeout) 147 })) 148 return conn, err 149} 150 151func newTestRuntimeClient(t *testing.T) runtime.RuntimeServiceClient { 152 ctx, cancel := context.WithTimeout(context.Background(), connectTimeout) 153 defer cancel() 154 conn, err := createGRPCConn(ctx) 155 if err != nil { 156 t.Fatalf("failed to dial runtime client: %v", err) 157 } 158 return runtime.NewRuntimeServiceClient(conn) 159} 160 161func newTestEventService(t *testing.T) containerd.EventService { 162 ctx, cancel := context.WithTimeout(context.Background(), connectTimeout) 163 defer cancel() 164 conn, err := createGRPCConn(ctx) 165 if err != nil { 166 t.Fatalf("Failed to create a client connection %v", err) 167 } 168 return containerd.NewEventServiceFromClient(eventsapi.NewEventsClient(conn)) 169} 170 171func newTestImageClient(t *testing.T) runtime.ImageServiceClient { 172 ctx, cancel := context.WithTimeout(context.Background(), connectTimeout) 173 defer cancel() 174 conn, err := createGRPCConn(ctx) 175 if err != nil { 176 t.Fatalf("failed to dial runtime client: %v", err) 177 } 178 return runtime.NewImageServiceClient(conn) 179} 180 181func getTargetRunTopics() (topicNames []string, filters []string) { 182 topicNames = []string{ 183 eventruntime.TaskCreateEventTopic, 184 eventruntime.TaskStartEventTopic, 185 eventruntime.TaskExitEventTopic, 186 eventruntime.TaskDeleteEventTopic, 187 } 188 189 filters = make([]string, len(topicNames)) 190 191 for i, name := range topicNames { 192 filters[i] = fmt.Sprintf(`topic=="%v"`, name) 193 } 194 return topicNames, filters 195} 196 197func convertEvent(e *types.Any) (string, interface{}, error) { 198 id := "" 199 evt, err := typeurl.UnmarshalAny(e) 200 if err != nil { 201 return "", nil, err 202 } 203 204 switch event := evt.(type) { 205 case *eventtypes.TaskCreate: 206 id = event.ContainerID 207 case *eventtypes.TaskStart: 208 id = event.ContainerID 209 case *eventtypes.TaskDelete: 210 id = event.ContainerID 211 case *eventtypes.TaskExit: 212 id = event.ContainerID 213 default: 214 return "", nil, errors.New("test does not support this event") 215 } 216 return id, evt, nil 217} 218 219func pullRequiredImages(t *testing.T, images []string) { 220 pullRequiredImagesWithLabels(t, images, map[string]string{ 221 "sandbox-platform": "windows/amd64", // Not required for Windows but makes the test safer depending on defaults in the config. 222 }) 223} 224 225func pullRequiredLcowImages(t *testing.T, images []string) { 226 pullRequiredImagesWithLabels(t, images, map[string]string{ 227 "sandbox-platform": "linux/amd64", 228 }) 229} 230 231func pullRequiredImagesWithLabels(t *testing.T, images []string, labels map[string]string) { 232 if len(images) < 1 { 233 return 234 } 235 236 client := newTestImageClient(t) 237 ctx, cancel := context.WithCancel(context.Background()) 238 defer cancel() 239 240 sb := &runtime.PodSandboxConfig{ 241 Labels: labels, 242 } 243 for _, image := range images { 244 _, err := client.PullImage(ctx, &runtime.PullImageRequest{ 245 Image: &runtime.ImageSpec{ 246 Image: image, 247 }, 248 SandboxConfig: sb, 249 }) 250 if err != nil { 251 t.Fatalf("failed PullImage for image: %s, with error: %v", image, err) 252 } 253 } 254} 255 256func removeImages(t *testing.T, images []string) { 257 if len(images) < 1 { 258 return 259 } 260 261 client := newTestImageClient(t) 262 ctx, cancel := context.WithCancel(context.Background()) 263 defer cancel() 264 265 for _, image := range images { 266 _, err := client.RemoveImage(ctx, &runtime.RemoveImageRequest{ 267 Image: &runtime.ImageSpec{ 268 Image: image, 269 }, 270 }) 271 if err != nil { 272 t.Fatalf("failed removeImage for image: %s, with error: %v", image, err) 273 } 274 } 275} 276