From af04d41e1b75248c4312b7d68053f9f7464dc888 Mon Sep 17 00:00:00 2001 From: Tuomas Katila Date: Mon, 8 Jan 2024 14:41:44 +0200 Subject: [PATCH 1/6] labeler: use a function to store splittable labels Signed-off-by: Tuomas Katila --- cmd/internal/labeler/labeler.go | 35 +++++++++++++++------------------ 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/cmd/internal/labeler/labeler.go b/cmd/internal/labeler/labeler.go index 869bd8da4..d5da04ebb 100644 --- a/cmd/internal/labeler/labeler.go +++ b/cmd/internal/labeler/labeler.go @@ -232,6 +232,19 @@ func (lm labelMap) addNumericLabel(labelName string, valueToAdd int64) { lm[labelName] = strconv.FormatInt(value, 10) } +// Stores a long string to labels so that it's possibly split into multiple +// keys: foobar="", foobar2="", foobar3="The end." +func (lm labelMap) addSplittableString(labelBase, fullValue string) { + splitList := pluginutils.SplitAtLastAlphaNum(fullValue, labelMaxLength, labelControlChar) + + lm[labelBase] = splitList[0] + + for i := 1; i < len(splitList); i++ { + nextLabel := labelBase + strconv.FormatInt(int64(i+1), 10) + lm[nextLabel] = splitList[i] + } +} + // this returns pci groups label value, groups separated by "_", gpus separated by ".". // Example for two groups with 4 gpus: "0.1.2.3_4.5.6.7". func (l *labeler) createPCIGroupLabel(gpuNumList []string) string { @@ -327,24 +340,13 @@ func (l *labeler) createLabels() error { strings.Join(gpuNameList, "."), labelMaxLength, labelControlChar)[0] // add gpu num list label(s) (example: "0.1.2", which is short form of "card0.card1.card2") - allGPUs := strings.Join(gpuNumList, ".") - gpuNumLists := pluginutils.SplitAtLastAlphaNum(allGPUs, labelMaxLength, labelControlChar) - - l.labels[labelNamespace+gpuNumListLabelName] = gpuNumLists[0] - for i := 1; i < len(gpuNumLists); i++ { - l.labels[labelNamespace+gpuNumListLabelName+strconv.FormatInt(int64(i+1), 10)] = gpuNumLists[i] - } + l.labels.addSplittableString(labelNamespace+gpuNumListLabelName, strings.Join(gpuNumList, ".")) if len(numaMapping) > 0 { // add numa node mapping to labels: gpu.intel.com/numa-gpu-map="0-0.1.2.3_1-4.5.6.7" numaMappingLabel := createNumaNodeMappingLabel(numaMapping) - numaMappingLabelList := pluginutils.SplitAtLastAlphaNum(numaMappingLabel, labelMaxLength, labelControlChar) - - l.labels[labelNamespace+numaMappingName] = numaMappingLabelList[0] - for i := 1; i < len(numaMappingLabelList); i++ { - l.labels[labelNamespace+numaMappingName+strconv.FormatInt(int64(i+1), 10)] = numaMappingLabelList[i] - } + l.labels.addSplittableString(labelNamespace+numaMappingName, numaMappingLabel) } // all GPUs get default number of millicores (1000) @@ -353,12 +355,7 @@ func (l *labeler) createLabels() error { // aa pci-group label(s), (two group example: "1.2.3.4_5.6.7.8") allPCIGroups := l.createPCIGroupLabel(gpuNumList) if allPCIGroups != "" { - pciGroups := pluginutils.SplitAtLastAlphaNum(allPCIGroups, labelMaxLength, labelControlChar) - - l.labels[labelNamespace+pciGroupLabelName] = pciGroups[0] - for i := 1; i < len(gpuNumLists); i++ { - l.labels[labelNamespace+pciGroupLabelName+strconv.FormatInt(int64(i+1), 10)] = pciGroups[i] - } + l.labels.addSplittableString(labelNamespace+pciGroupLabelName, allPCIGroups) } } From d5cb53a1d1a5b3a9522dc903d19aa82985ec106c Mon Sep 17 00:00:00 2001 From: Tuomas Katila Date: Mon, 8 Jan 2024 14:42:54 +0200 Subject: [PATCH 2/6] labeler: add xe support for tile counting Signed-off-by: Tuomas Katila --- cmd/internal/labeler/labeler.go | 14 ++- cmd/internal/labeler/labeler_test.go | 122 +++++++++++++++------------ 2 files changed, 78 insertions(+), 58 deletions(-) diff --git a/cmd/internal/labeler/labeler.go b/cmd/internal/labeler/labeler.go index d5da04ebb..0d2fdc19f 100644 --- a/cmd/internal/labeler/labeler.go +++ b/cmd/internal/labeler/labeler.go @@ -184,10 +184,16 @@ func GetMemoryAmount(sysfsDrmDir, gpuName string, numTiles uint64) uint64 { } // GetTileCount reads the tile count. -func GetTileCount(sysfsDrmDir, gpuName string) (numTiles uint64) { - filePath := filepath.Join(sysfsDrmDir, gpuName, "gt/gt*") +func GetTileCount(cardPath string) (numTiles uint64) { + files := []string{} - files, _ := filepath.Glob(filePath) + paths, _ := filepath.Glob(filepath.Join(cardPath, "gt/gt*")) // i915 driver + files = append(files, paths...) + + paths, _ = filepath.Glob(filepath.Join(cardPath, "device/tile?")) // Xe driver + files = append(files, paths...) + + klog.V(4).Info("tile files found:", files) if len(files) == 0 { return 1 @@ -308,7 +314,7 @@ func (l *labeler) createLabels() error { return errors.Wrap(err, "gpu name parsing error") } - numTiles := GetTileCount(l.sysfsDRMDir, gpuName) + numTiles := GetTileCount(filepath.Join(l.sysfsDRMDir, gpuName)) tileCount += int(numTiles) memoryAmount := GetMemoryAmount(l.sysfsDRMDir, gpuName, numTiles) diff --git a/cmd/internal/labeler/labeler_test.go b/cmd/internal/labeler/labeler_test.go index 31186e224..e3dd50bef 100644 --- a/cmd/internal/labeler/labeler_test.go +++ b/cmd/internal/labeler/labeler_test.go @@ -137,60 +137,6 @@ func getTestCases() []testcase { "gpu.intel.com/tiles": "1", }, }, - { - sysfsdirs: []string{ - "card0/device/drm/card0", - }, - sysfsfiles: map[string][]byte{ - "card0/device/vendor": []byte("0x8086"), - }, - name: "when gen:capability info is missing", - memoryOverride: 16000000000, - expectedRetval: nil, - expectedLabels: labelMap{ - "gpu.intel.com/millicores": "1000", - "gpu.intel.com/memory.max": "16000000000", - "gpu.intel.com/cards": "card0", - "gpu.intel.com/gpu-numbers": "0", - "gpu.intel.com/tiles": "1", - }, - }, - { - sysfsdirs: []string{ - "card0/device/drm/card0", - }, - sysfsfiles: map[string][]byte{ - "card0/device/vendor": []byte("0x8086"), - }, - name: "gen version missing, but media & graphics versions present", - memoryOverride: 16000000000, - expectedRetval: nil, - expectedLabels: labelMap{ - "gpu.intel.com/millicores": "1000", - "gpu.intel.com/memory.max": "16000000000", - "gpu.intel.com/cards": "card0", - "gpu.intel.com/gpu-numbers": "0", - "gpu.intel.com/tiles": "1", - }, - }, - { - sysfsdirs: []string{ - "card0/device/drm/card0", - }, - sysfsfiles: map[string][]byte{ - "card0/device/vendor": []byte("0x8086"), - }, - name: "only media version present", - memoryOverride: 16000000000, - expectedRetval: nil, - expectedLabels: labelMap{ - "gpu.intel.com/millicores": "1000", - "gpu.intel.com/memory.max": "16000000000", - "gpu.intel.com/cards": "card0", - "gpu.intel.com/gpu-numbers": "0", - "gpu.intel.com/tiles": "1", - }, - }, { sysfsdirs: []string{ "card0/device/drm/card0", @@ -562,6 +508,74 @@ func getTestCases() []testcase { "gpu.intel.com/tiles": "1", }, }, + { + sysfsdirs: []string{ + "card0/device/drm/card0", + "card0/device/tile0/gt0", + "card0/device/tile1/gt0", + "card1/device/drm/card1", + "card1/device/tile0/gt0", + "card1/device/tile1/gt0", + "card2/device/drm/card2", + "card2/device/tile0/gt0", + "card2/device/tile1/gt0", + }, + sysfsfiles: map[string][]byte{ + "card0/device/vendor": []byte("0x8086"), + "card0/lmem_total_bytes": []byte("8000"), + "card0/device/numa_node": []byte("1"), + "card1/device/vendor": []byte("0x8086"), + "card1/lmem_total_bytes": []byte("8000"), + "card1/device/numa_node": []byte("1"), + "card2/device/vendor": []byte("0x8086"), + "card2/lmem_total_bytes": []byte("8000"), + "card2/device/numa_node": []byte("1"), + }, + name: "successful labeling with three cards and with xe driver", + expectedRetval: nil, + expectedLabels: labelMap{ + "gpu.intel.com/millicores": "3000", + "gpu.intel.com/memory.max": "48000", + "gpu.intel.com/gpu-numbers": "0.1.2", + "gpu.intel.com/cards": "card0.card1.card2", + "gpu.intel.com/tiles": "6", + "gpu.intel.com/numa-gpu-map": "1-0.1.2", + }, + }, + { + sysfsdirs: []string{ + "card0/device/drm/card0", + "card0/device/tile0/gt0", + "card0/device/tile0/gt1", + "card0/device/tile1/gt2", + "card0/device/tile1/gt3", + "card0/device/tile1/gt4", + "card0/device/tile1/gt5", + "card1/device/drm/card1", + "card1/device/tile0/gt0", + "card1/device/tile0/gt1", + "card1/device/tile1/gt2", + "card1/device/tile1/gt4", + }, + sysfsfiles: map[string][]byte{ + "card0/device/vendor": []byte("0x8086"), + "card0/lmem_total_bytes": []byte("8000"), + "card0/device/numa_node": []byte("1"), + "card1/device/vendor": []byte("0x8086"), + "card1/lmem_total_bytes": []byte("8000"), + "card1/device/numa_node": []byte("1"), + }, + name: "successful labeling with two cards, two tiles per card and multiple gts per tile", + expectedRetval: nil, + expectedLabels: labelMap{ + "gpu.intel.com/millicores": "2000", + "gpu.intel.com/memory.max": "32000", + "gpu.intel.com/gpu-numbers": "0.1", + "gpu.intel.com/cards": "card0.card1", + "gpu.intel.com/tiles": "4", + "gpu.intel.com/numa-gpu-map": "1-0.1", + }, + }, } } From e600fe93136b16ab9320cb24a67e92664e4e6bb0 Mon Sep 17 00:00:00 2001 From: Tuomas Katila Date: Mon, 8 Jan 2024 15:04:33 +0200 Subject: [PATCH 3/6] gpu: add support for the upcoming xe-driver Plugin can support both i915 and xe drivers dynamically. But having both drivers on same node with RM is not possible. Signed-off-by: Tuomas Katila --- .github/workflows/lib-e2e.yaml | 1 + cmd/gpu_plugin/device_props.go | 85 +++++ cmd/gpu_plugin/gpu_plugin.go | 162 ++++++--- cmd/gpu_plugin/gpu_plugin_test.go | 317 +++++++++++++++--- .../rm/gpu_plugin_resource_manager.go | 80 ++--- .../rm/gpu_plugin_resource_manager_test.go | 22 +- cmd/internal/pluginutils/devicedriver.go | 30 ++ cmd/internal/pluginutils/devicedriver_test.go | 80 +++++ test/e2e/gpu/gpu.go | 53 +++ 9 files changed, 672 insertions(+), 158 deletions(-) create mode 100644 cmd/gpu_plugin/device_props.go create mode 100644 cmd/internal/pluginutils/devicedriver.go create mode 100644 cmd/internal/pluginutils/devicedriver_test.go diff --git a/.github/workflows/lib-e2e.yaml b/.github/workflows/lib-e2e.yaml index 057c03687..1678243be 100644 --- a/.github/workflows/lib-e2e.yaml +++ b/.github/workflows/lib-e2e.yaml @@ -25,6 +25,7 @@ jobs: - name: e2e-gpu runner: gpu images: intel-gpu-plugin intel-gpu-initcontainer + targetJob: e2e-gpu SKIP=Resource:xe - name: e2e-iaa-spr targetjob: e2e-iaa runner: simics-spr diff --git a/cmd/gpu_plugin/device_props.go b/cmd/gpu_plugin/device_props.go new file mode 100644 index 000000000..e6daf2f28 --- /dev/null +++ b/cmd/gpu_plugin/device_props.go @@ -0,0 +1,85 @@ +// Copyright 2024 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "slices" + + "github.com/intel/intel-device-plugins-for-kubernetes/cmd/internal/labeler" + "github.com/intel/intel-device-plugins-for-kubernetes/cmd/internal/pluginutils" + "k8s.io/klog/v2" +) + +type DeviceProperties struct { + currentDriver string + drmDrivers map[string]bool + tileCounts []uint64 + isPfWithVfs bool +} + +type invalidTileCountErr struct { + error +} + +func newDeviceProperties() *DeviceProperties { + return &DeviceProperties{ + drmDrivers: make(map[string]bool), + } +} + +func (d *DeviceProperties) fetch(cardPath string) { + d.isPfWithVfs = pluginutils.IsSriovPFwithVFs(cardPath) + + d.tileCounts = append(d.tileCounts, labeler.GetTileCount(cardPath)) + + driverName, err := pluginutils.ReadDeviceDriver(cardPath) + if err != nil { + klog.Warningf("card (%s) doesn't have driver, using default: %s", cardPath, deviceTypeDefault) + + driverName = deviceTypeDefault + } + + d.currentDriver = driverName + d.drmDrivers[d.currentDriver] = true +} + +func (d *DeviceProperties) drmDriverCount() int { + return len(d.drmDrivers) +} + +func (d *DeviceProperties) driver() string { + return d.currentDriver +} + +func (d *DeviceProperties) monitorResource() string { + return d.currentDriver + monitorSuffix +} + +func (d *DeviceProperties) maxTileCount() (uint64, error) { + if len(d.tileCounts) == 0 { + return 0, invalidTileCountErr{} + } + + minCount := slices.Min(d.tileCounts) + maxCount := slices.Max(d.tileCounts) + + if minCount != maxCount { + klog.Warningf("Node's GPUs are heterogenous (min: %d, max: %d tiles)", minCount, maxCount) + + return 0, invalidTileCountErr{} + } + + return maxCount, nil +} diff --git a/cmd/gpu_plugin/gpu_plugin.go b/cmd/gpu_plugin/gpu_plugin.go index 6f1ad4018..44c504263 100644 --- a/cmd/gpu_plugin/gpu_plugin.go +++ b/cmd/gpu_plugin/gpu_plugin.go @@ -17,6 +17,7 @@ package main import ( "flag" "fmt" + "io/fs" "os" "path" "path/filepath" @@ -32,7 +33,6 @@ import ( "github.com/intel/intel-device-plugins-for-kubernetes/cmd/gpu_plugin/rm" "github.com/intel/intel-device-plugins-for-kubernetes/cmd/internal/labeler" - "github.com/intel/intel-device-plugins-for-kubernetes/cmd/internal/pluginutils" dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin" ) @@ -47,12 +47,14 @@ const ( vendorString = "0x8086" // Device plugin settings. - namespace = "gpu.intel.com" - deviceType = "i915" + namespace = "gpu.intel.com" + deviceTypeI915 = "i915" + deviceTypeXe = "xe" + deviceTypeDefault = deviceTypeI915 // telemetry resource settings. - monitorType = "i915_monitoring" - monitorID = "all" + monitorSuffix = "_monitoring" + monitorID = "all" // Period of device scans. scanPeriod = 5 * time.Second @@ -68,6 +70,10 @@ type cliOptions struct { resourceManagement bool } +type rmWithMultipleDriversErr struct { + error +} + type preferredAllocationPolicyFunc func(*pluginapi.ContainerPreferredAllocationRequest) []string // nonePolicy is used for allocating GPU devices randomly, while trying @@ -283,7 +289,11 @@ func newDevicePlugin(sysfsDir, devfsDir string, options cliOptions) *devicePlugi if options.resourceManagement { var err error - dp.resMan, err = rm.NewResourceManager(monitorID, namespace+"/"+deviceType) + dp.resMan, err = rm.NewResourceManager(monitorID, + []string{ + namespace + "/" + deviceTypeI915, + namespace + "/" + deviceTypeXe, + }) if err != nil { klog.Errorf("Failed to create resource manager: %+v", err) return nil @@ -345,13 +355,20 @@ func (dp *devicePlugin) GetPreferredAllocation(rqt *pluginapi.PreferredAllocatio func (dp *devicePlugin) Scan(notifier dpapi.Notifier) error { defer dp.scanTicker.Stop() - klog.V(1).Infof("GPU '%s' resource share count = %d", deviceType, dp.options.sharedDevNum) + klog.V(1).Infof("GPU (%s/%s) resource share count = %d", deviceTypeI915, deviceTypeXe, dp.options.sharedDevNum) - previousCount := map[string]int{deviceType: 0, monitorType: 0} + previousCount := map[string]int{ + deviceTypeI915: 0, deviceTypeXe: 0, + deviceTypeXe + monitorSuffix: 0, + deviceTypeI915 + monitorSuffix: 0} for { devTree, err := dp.scan() if err != nil { + if errors.Is(err, rmWithMultipleDriversErr{}) { + return err + } + klog.Warning("Failed to scan: ", err) } @@ -426,81 +443,116 @@ func (dp *devicePlugin) devSpecForDrmFile(drmFile string) (devSpec pluginapi.Dev return } +func (dp *devicePlugin) filterOutInvalidCards(files []fs.DirEntry) []fs.DirEntry { + filtered := []fs.DirEntry{} + + for _, f := range files { + if !dp.isCompatibleDevice(f.Name()) { + continue + } + + _, err := os.Stat(path.Join(dp.sysfsDir, f.Name(), "device/drm")) + if err != nil { + continue + } + + filtered = append(filtered, f) + } + + return filtered +} + +func (dp *devicePlugin) createDeviceSpecsFromDrmFiles(cardPath string) []pluginapi.DeviceSpec { + specs := []pluginapi.DeviceSpec{} + + drmFiles, _ := os.ReadDir(path.Join(cardPath, "device/drm")) + + for _, drmFile := range drmFiles { + devSpec, devPath, devSpecErr := dp.devSpecForDrmFile(drmFile.Name()) + if devSpecErr != nil { + continue + } + + klog.V(4).Infof("Adding %s to GPU %s", devPath, filepath.Base(cardPath)) + + specs = append(specs, devSpec) + } + + return specs +} + func (dp *devicePlugin) scan() (dpapi.DeviceTree, error) { files, err := os.ReadDir(dp.sysfsDir) if err != nil { return nil, errors.Wrap(err, "Can't read sysfs folder") } - var monitor []pluginapi.DeviceSpec + monitor := make(map[string][]pluginapi.DeviceSpec, 0) devTree := dpapi.NewDeviceTree() rmDevInfos := rm.NewDeviceInfoMap() - tileCounts := []uint64{} + devProps := newDeviceProperties() - for _, f := range files { - var nodes []pluginapi.DeviceSpec + for _, f := range dp.filterOutInvalidCards(files) { + name := f.Name() + cardPath := path.Join(dp.sysfsDir, name) - if !dp.isCompatibleDevice(f.Name()) { + devProps.fetch(cardPath) + + if devProps.isPfWithVfs { continue } - cardPath := path.Join(dp.sysfsDir, f.Name()) + devSpecs := dp.createDeviceSpecsFromDrmFiles(cardPath) - drmFiles, err := os.ReadDir(path.Join(cardPath, "device/drm")) - if err != nil { - return nil, errors.Wrap(err, "Can't read device folder") + if len(devSpecs) == 0 { + continue } - isPFwithVFs := pluginutils.IsSriovPFwithVFs(path.Join(dp.sysfsDir, f.Name())) - tileCounts = append(tileCounts, labeler.GetTileCount(dp.sysfsDir, f.Name())) - - for _, drmFile := range drmFiles { - devSpec, devPath, devSpecErr := dp.devSpecForDrmFile(drmFile.Name()) - if devSpecErr != nil { - continue - } - - if !isPFwithVFs { - klog.V(4).Infof("Adding %s to GPU %s", devPath, f.Name()) + mounts := []pluginapi.Mount{} + if dp.bypathFound { + mounts = dp.bypathMountsForPci(cardPath, name, dp.bypathDir) + } - nodes = append(nodes, devSpec) - } + deviceInfo := dpapi.NewDeviceInfo(pluginapi.Healthy, devSpecs, mounts, nil, nil) - if dp.options.enableMonitoring { - klog.V(4).Infof("Adding %s to GPU %s/%s", devPath, monitorType, monitorID) + for i := 0; i < dp.options.sharedDevNum; i++ { + devID := fmt.Sprintf("%s-%d", name, i) + devTree.AddDevice(devProps.driver(), devID, deviceInfo) - monitor = append(monitor, devSpec) - } + rmDevInfos[devID] = rm.NewDeviceInfo(devSpecs, mounts, nil) } - if len(nodes) > 0 { - mounts := []pluginapi.Mount{} - if dp.bypathFound { - mounts = dp.bypathMountsForPci(cardPath, f.Name(), dp.bypathDir) - } - - deviceInfo := dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, mounts, nil, nil) - - for i := 0; i < dp.options.sharedDevNum; i++ { - devID := fmt.Sprintf("%s-%d", f.Name(), i) - // Currently only one device type (i915) is supported. - // TODO: check model ID to differentiate device models. - devTree.AddDevice(deviceType, devID, deviceInfo) + if dp.options.enableMonitoring { + res := devProps.monitorResource() + klog.V(4).Infof("For %s/%s, adding nodes: %+v", res, monitorID, devSpecs) - rmDevInfos[devID] = rm.NewDeviceInfo(nodes, mounts, nil) - } + monitor[res] = append(monitor[res], devSpecs...) } } - // all Intel GPUs are under single monitoring resource + + // all Intel GPUs are under single monitoring resource per KMD if len(monitor) > 0 { - deviceInfo := dpapi.NewDeviceInfo(pluginapi.Healthy, monitor, nil, nil, nil) - devTree.AddDevice(monitorType, monitorID, deviceInfo) + for resourceName, devices := range monitor { + deviceInfo := dpapi.NewDeviceInfo(pluginapi.Healthy, devices, nil, nil, nil) + devTree.AddDevice(resourceName, monitorID, deviceInfo) + } } if dp.resMan != nil { - dp.resMan.SetDevInfos(rmDevInfos) - dp.resMan.SetTileCountPerCard(tileCounts) + if devProps.drmDriverCount() <= 1 { + dp.resMan.SetDevInfos(rmDevInfos) + + if tileCount, err := devProps.maxTileCount(); err == nil { + dp.resMan.SetTileCountPerCard(tileCount) + } + } else { + klog.Warning("Plugin with RM doesn't support multiple DRM drivers:", devProps.drmDrivers) + + err := rmWithMultipleDriversErr{} + + return nil, err + } } return devTree, nil @@ -521,7 +573,7 @@ func main() { ) flag.StringVar(&prefix, "prefix", "", "Prefix for devfs & sysfs paths") - flag.BoolVar(&opts.enableMonitoring, "enable-monitoring", false, "whether to enable 'i915_monitoring' (= all GPUs) resource") + flag.BoolVar(&opts.enableMonitoring, "enable-monitoring", false, "whether to enable '*_monitoring' (= all GPUs) resource") flag.BoolVar(&opts.resourceManagement, "resource-manager", false, "fractional GPU resource management") flag.IntVar(&opts.sharedDevNum, "shared-dev-num", 1, "number of containers sharing the same GPU device") flag.StringVar(&opts.preferredAllocationPolicy, "allocation-policy", "none", "modes of allocating GPU devices: balanced, packed and none") diff --git a/cmd/gpu_plugin/gpu_plugin_test.go b/cmd/gpu_plugin/gpu_plugin_test.go index 0277a089f..e0ecd6b24 100644 --- a/cmd/gpu_plugin/gpu_plugin_test.go +++ b/cmd/gpu_plugin/gpu_plugin_test.go @@ -37,20 +37,26 @@ func init() { // mockNotifier implements Notifier interface. type mockNotifier struct { - scanDone chan bool - devCount int - monitorCount int + scanDone chan bool + i915Count int + xeCount int + i915monitorCount int + xeMonitorCount int } // Notify stops plugin Scan. func (n *mockNotifier) Notify(newDeviceTree dpapi.DeviceTree) { - n.monitorCount = len(newDeviceTree[monitorType]) - n.devCount = len(newDeviceTree[deviceType]) + n.xeCount = len(newDeviceTree[deviceTypeXe]) + n.xeMonitorCount = len(newDeviceTree[deviceTypeXe+monitorSuffix]) + n.i915Count = len(newDeviceTree[deviceTypeI915]) + n.i915monitorCount = len(newDeviceTree[deviceTypeDefault+monitorSuffix]) n.scanDone <- true } -type mockResourceManager struct{} +type mockResourceManager struct { + tileCount uint64 +} func (m *mockResourceManager) CreateFractionalResourceResponse(*v1beta1.AllocateRequest) (*v1beta1.AllocateResponse, error) { return &v1beta1.AllocateResponse{}, &dpapi.UseDefaultMethodError{} @@ -61,31 +67,62 @@ func (m *mockResourceManager) GetPreferredFractionalAllocation(*v1beta1.Preferre return &v1beta1.PreferredAllocationResponse{}, &dpapi.UseDefaultMethodError{} } -func (m *mockResourceManager) SetTileCountPerCard(counts []uint64) { +func (m *mockResourceManager) SetTileCountPerCard(count uint64) { + m.tileCount = count +} + +type TestCaseDetails struct { + name string + // test-case environment + sysfsdirs []string + sysfsfiles map[string][]byte + symlinkfiles map[string]string + devfsdirs []string + // how plugin should interpret it + options cliOptions + // what the result should be (i915) + expectedI915Devs int + expectedI915Monitors int + // what the result should be (xe) + expectedXeDevs int + expectedXeMonitors int } -func createTestFiles(root string, devfsdirs, sysfsdirs []string, sysfsfiles map[string][]byte) (string, string, error) { +func createTestFiles(root string, tc TestCaseDetails) (string, string, error) { sysfs := path.Join(root, "sys") devfs := path.Join(root, "dev") - for _, devfsdir := range devfsdirs { + for _, devfsdir := range tc.devfsdirs { if err := os.MkdirAll(path.Join(devfs, devfsdir), 0750); err != nil { return "", "", errors.Wrap(err, "Failed to create fake device directory") } } - for _, sysfsdir := range sysfsdirs { + for _, sysfsdir := range tc.sysfsdirs { if err := os.MkdirAll(path.Join(sysfs, sysfsdir), 0750); err != nil { return "", "", errors.Wrap(err, "Failed to create fake device directory") } } - for filename, body := range sysfsfiles { + for filename, body := range tc.sysfsfiles { if err := os.WriteFile(path.Join(sysfs, filename), body, 0600); err != nil { return "", "", errors.Wrap(err, "Failed to create fake vendor file") } } + for source, target := range tc.symlinkfiles { + driverPath := path.Join(sysfs, target) + symlinkPath := path.Join(sysfs, source) + + if err := os.MkdirAll(driverPath, 0750); err != nil { + return "", "", errors.Wrap(err, "Failed to create fake driver file.") + } + + if err := os.Symlink(driverPath, symlinkPath); err != nil { + return "", "", errors.Wrap(err, "Failed to create fake driver symlink file.") + } + } + return sysfs, devfs, nil } @@ -186,18 +223,7 @@ func TestAllocate(t *testing.T) { } func TestScan(t *testing.T) { - tcases := []struct { - name string - // test-case environment - sysfsdirs []string - sysfsfiles map[string][]byte - devfsdirs []string - // how plugin should interpret it - options cliOptions - // what the result should be - expectedDevs int - expectedMonitors int - }{ + tcases := []TestCaseDetails{ { name: "no sysfs mounted", }, @@ -223,7 +249,71 @@ func TestScan(t *testing.T) { "by-path/pci-0000:00:00.0-card", "by-path/pci-0000:00:00.0-render", }, - expectedDevs: 1, + expectedI915Devs: 1, + }, + { + name: "one device with xe driver", + sysfsdirs: []string{"card0/device/drm/card0", "card0/device/drm/controlD64"}, + sysfsfiles: map[string][]byte{ + "card0/device/vendor": []byte("0x8086"), + }, + symlinkfiles: map[string]string{ + "card0/device/driver": "drivers/xe", + }, + devfsdirs: []string{ + "card0", + "by-path/pci-0000:00:00.0-card", + "by-path/pci-0000:00:00.0-render", + }, + expectedXeDevs: 1, + }, + { + name: "two devices with xe driver and monitoring", + sysfsdirs: []string{"card0/device/drm/card0", "card0/device/drm/controlD64", "card1/device/drm/card1"}, + sysfsfiles: map[string][]byte{ + "card0/device/vendor": []byte("0x8086"), + "card1/device/vendor": []byte("0x8086"), + }, + symlinkfiles: map[string]string{ + "card0/device/driver": "drivers/xe", + "card1/device/driver": "drivers/xe", + }, + devfsdirs: []string{ + "card0", + "by-path/pci-0000:00:00.0-card", + "by-path/pci-0000:00:00.0-render", + "card1", + "by-path/pci-0000:00:01.0-card", + "by-path/pci-0000:00:01.0-render", + }, + options: cliOptions{enableMonitoring: true}, + expectedXeDevs: 2, + expectedXeMonitors: 1, + }, + { + name: "two devices with xe and i915 drivers", + sysfsdirs: []string{"card0/device/drm/card0", "card0/device/drm/controlD64", "card1/device/drm/card1"}, + sysfsfiles: map[string][]byte{ + "card0/device/vendor": []byte("0x8086"), + "card1/device/vendor": []byte("0x8086"), + }, + symlinkfiles: map[string]string{ + "card0/device/driver": "drivers/xe", + "card1/device/driver": "drivers/i915", + }, + devfsdirs: []string{ + "card0", + "by-path/pci-0000:00:00.0-card", + "by-path/pci-0000:00:00.0-render", + "card1", + "by-path/pci-0000:00:01.0-card", + "by-path/pci-0000:00:01.0-render", + }, + options: cliOptions{enableMonitoring: true}, + expectedXeDevs: 1, + expectedXeMonitors: 1, + expectedI915Devs: 1, + expectedI915Monitors: 1, }, { name: "sriov-1-pf-no-vfs + monitoring", @@ -232,10 +322,10 @@ func TestScan(t *testing.T) { "card0/device/vendor": []byte("0x8086"), "card0/device/sriov_numvfs": []byte("0"), }, - devfsdirs: []string{"card0"}, - options: cliOptions{enableMonitoring: true}, - expectedDevs: 1, - expectedMonitors: 1, + devfsdirs: []string{"card0"}, + options: cliOptions{enableMonitoring: true}, + expectedI915Devs: 1, + expectedI915Monitors: 1, }, { name: "two sysfs records but one dev node", @@ -247,8 +337,8 @@ func TestScan(t *testing.T) { "card0/device/vendor": []byte("0x8086"), "card1/device/vendor": []byte("0x8086"), }, - devfsdirs: []string{"card0"}, - expectedDevs: 1, + devfsdirs: []string{"card0"}, + expectedI915Devs: 1, }, { name: "sriov-1-pf-and-2-vfs", @@ -263,8 +353,8 @@ func TestScan(t *testing.T) { "card1/device/vendor": []byte("0x8086"), "card2/device/vendor": []byte("0x8086"), }, - devfsdirs: []string{"card0", "card1", "card2"}, - expectedDevs: 2, + devfsdirs: []string{"card0", "card1", "card2"}, + expectedI915Devs: 2, }, { name: "two devices with 13 shares + monitoring", @@ -276,10 +366,10 @@ func TestScan(t *testing.T) { "card0/device/vendor": []byte("0x8086"), "card1/device/vendor": []byte("0x8086"), }, - devfsdirs: []string{"card0", "card1"}, - options: cliOptions{sharedDevNum: 13, enableMonitoring: true}, - expectedDevs: 26, - expectedMonitors: 1, + devfsdirs: []string{"card0", "card1"}, + options: cliOptions{sharedDevNum: 13, enableMonitoring: true}, + expectedI915Devs: 26, + expectedI915Monitors: 1, }, { name: "wrong vendor", @@ -317,7 +407,7 @@ func TestScan(t *testing.T) { // dirs/files need to be removed for the next test defer os.RemoveAll(root) - sysfs, devfs, err := createTestFiles(root, tc.devfsdirs, tc.sysfsdirs, tc.sysfsfiles) + sysfs, devfs, err := createTestFiles(root, tc) if err != nil { t.Errorf("unexpected error: %+v", err) } @@ -328,20 +418,157 @@ func TestScan(t *testing.T) { scanDone: plugin.scanDone, } - plugin.resMan = &mockResourceManager{} - err = plugin.Scan(notifier) // Scans in GPU plugin never fail if err != nil { t.Errorf("unexpected error: %+v", err) } - if tc.expectedDevs != notifier.devCount { - t.Errorf("Expected %d, discovered %d devices", - tc.expectedDevs, notifier.devCount) + if tc.expectedI915Devs != notifier.i915Count { + t.Errorf("Expected %d, discovered %d devices (i915)", + tc.expectedI915Devs, notifier.i915Count) + } + if tc.expectedI915Monitors != notifier.i915monitorCount { + t.Errorf("Expected %d, discovered %d monitors (i915)", + tc.expectedI915Monitors, notifier.i915monitorCount) + } + if tc.expectedXeDevs != notifier.xeCount { + t.Errorf("Expected %d, discovered %d devices (XE)", + tc.expectedXeDevs, notifier.xeCount) + } + if tc.expectedXeMonitors != notifier.xeMonitorCount { + t.Errorf("Expected %d, discovered %d monitors (XE)", + tc.expectedXeMonitors, notifier.xeMonitorCount) + } + }) + } +} + +func TestScanFails(t *testing.T) { + tc := TestCaseDetails{ + name: "xe and i915 devices with rm will fail", + sysfsdirs: []string{"card0/device/drm/card0", "card0/device/drm/controlD64", "card1/device/drm/card1"}, + sysfsfiles: map[string][]byte{ + "card0/device/vendor": []byte("0x8086"), + "card1/device/vendor": []byte("0x8086"), + }, + symlinkfiles: map[string]string{ + "card0/device/driver": "drivers/xe", + "card1/device/driver": "drivers/i915", + }, + devfsdirs: []string{ + "card0", + "card1", + }, + } + + t.Run(tc.name, func(t *testing.T) { + root, err := os.MkdirTemp("", "test_new_device_plugin") + if err != nil { + t.Fatalf("can't create temporary directory: %+v", err) + } + // dirs/files need to be removed for the next test + defer os.RemoveAll(root) + + sysfs, devfs, err := createTestFiles(root, tc) + if err != nil { + t.Errorf("unexpected error: %+v", err) + } + + plugin := newDevicePlugin(sysfs, devfs, tc.options) + + plugin.resMan = &mockResourceManager{} + + notifier := &mockNotifier{ + scanDone: plugin.scanDone, + } + + err = plugin.Scan(notifier) + if err == nil { + t.Error("unexpected nil error") + } + }) +} + +func TestScanWithRmAndTiles(t *testing.T) { + tcs := []TestCaseDetails{ + { + name: "two tile xe devices with rm enabled - homogeneous", + sysfsdirs: []string{ + "card0/device/drm/card0", + "card1/device/drm/card1", + "card0/device/tile0/gt0", + "card0/device/tile1/gt1", + "card1/device/tile0/gt0", + "card1/device/tile1/gt1", + }, + sysfsfiles: map[string][]byte{ + "card0/device/vendor": []byte("0x8086"), + "card1/device/vendor": []byte("0x8086"), + }, + symlinkfiles: map[string]string{ + "card0/device/driver": "drivers/xe", + "card1/device/driver": "drivers/xe", + }, + devfsdirs: []string{ + "card0", + "card1", + }, + }, + { + name: "2 & 1 tile xe devices with rm enabled - heterogeneous", + sysfsdirs: []string{ + "card0/device/drm/card0", + "card1/device/drm/card1", + "card0/device/tile0/gt0", + "card0/device/tile1/gt1", + "card1/device/tile0/gt0", + }, + sysfsfiles: map[string][]byte{ + "card0/device/vendor": []byte("0x8086"), + "card1/device/vendor": []byte("0x8086"), + }, + symlinkfiles: map[string]string{ + "card0/device/driver": "drivers/xe", + "card1/device/driver": "drivers/xe", + }, + devfsdirs: []string{ + "card0", + "card1", + }, + }, + } + + expectedTileCounts := []uint64{2, 0} + + for i, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + root, err := os.MkdirTemp("", "test_new_device_plugin") + if err != nil { + t.Fatalf("can't create temporary directory: %+v", err) + } + // dirs/files need to be removed for the next test + defer os.RemoveAll(root) + + sysfs, devfs, err := createTestFiles(root, tc) + if err != nil { + t.Errorf("unexpected error: %+v", err) + } + + plugin := newDevicePlugin(sysfs, devfs, tc.options) + + rm := &mockResourceManager{} + plugin.resMan = rm + + notifier := &mockNotifier{ + scanDone: plugin.scanDone, + } + + err = plugin.Scan(notifier) + if err != nil { + t.Error("unexpected error") } - if tc.expectedMonitors != notifier.monitorCount { - t.Errorf("Expected %d, discovered %d monitors", - tc.expectedMonitors, notifier.monitorCount) + if rm.tileCount != expectedTileCounts[i] { + t.Error("unexpected tilecount for RM") } }) } diff --git a/cmd/gpu_plugin/rm/gpu_plugin_resource_manager.go b/cmd/gpu_plugin/rm/gpu_plugin_resource_manager.go index 491d27fe1..4a5046da0 100644 --- a/cmd/gpu_plugin/rm/gpu_plugin_resource_manager.go +++ b/cmd/gpu_plugin/rm/gpu_plugin_resource_manager.go @@ -25,7 +25,6 @@ import ( "net" "net/http" "os" - "slices" "sort" "strconv" "strings" @@ -105,7 +104,7 @@ type ResourceManager interface { CreateFractionalResourceResponse(*pluginapi.AllocateRequest) (*pluginapi.AllocateResponse, error) GetPreferredFractionalAllocation(*pluginapi.PreferredAllocationRequest) (*pluginapi.PreferredAllocationResponse, error) SetDevInfos(DeviceInfoMap) - SetTileCountPerCard(counts []uint64) + SetTileCountPerCard(count uint64) } type containerAssignments struct { @@ -118,20 +117,20 @@ type podAssignmentDetails struct { } type resourceManager struct { - clientset kubernetes.Interface - deviceInfos DeviceInfoMap - prGetClientFunc getClientFunc - assignments map[string]podAssignmentDetails // pod name -> assignment details - nodeName string - hostIP string - skipID string - fullResourceName string - retryTimeout time.Duration - cleanupInterval time.Duration - mutex sync.RWMutex // for devTree updates during scan - cleanupMutex sync.RWMutex // for assignment details during cleanup - useKubelet bool - tileCountPerCard uint64 + clientset kubernetes.Interface + deviceInfos DeviceInfoMap + prGetClientFunc getClientFunc + assignments map[string]podAssignmentDetails // pod name -> assignment details + nodeName string + hostIP string + skipID string + fullResourceNames []string + retryTimeout time.Duration + cleanupInterval time.Duration + mutex sync.RWMutex // for devTree updates during scan + cleanupMutex sync.RWMutex // for assignment details during cleanup + useKubelet bool + tileCountPerCard uint64 } // NewDeviceInfo creates a new DeviceInfo. @@ -152,7 +151,7 @@ func NewDeviceInfoMap() DeviceInfoMap { } // NewResourceManager creates a new resource manager. -func NewResourceManager(skipID, fullResourceName string) (ResourceManager, error) { +func NewResourceManager(skipID string, fullResourceNames []string) (ResourceManager, error) { clientset, err := getClientset() if err != nil { @@ -160,16 +159,16 @@ func NewResourceManager(skipID, fullResourceName string) (ResourceManager, error } rm := resourceManager{ - nodeName: os.Getenv("NODE_NAME"), - hostIP: os.Getenv("HOST_IP"), - clientset: clientset, - skipID: skipID, - fullResourceName: fullResourceName, - prGetClientFunc: podresources.GetV1Client, - assignments: make(map[string]podAssignmentDetails), - retryTimeout: 1 * time.Second, - cleanupInterval: 20 * time.Minute, - useKubelet: true, + nodeName: os.Getenv("NODE_NAME"), + hostIP: os.Getenv("HOST_IP"), + clientset: clientset, + skipID: skipID, + fullResourceNames: fullResourceNames, + prGetClientFunc: podresources.GetV1Client, + assignments: make(map[string]podAssignmentDetails), + retryTimeout: 1 * time.Second, + cleanupInterval: 20 * time.Minute, + useKubelet: true, } klog.Info("GPU device plugin resource manager enabled") @@ -684,7 +683,7 @@ func (rm *resourceManager) getNodePendingGPUPods() (map[string]*v1.Pod, error) { pendingPods := rm.listPodsOnNodeWithStates([]string{string(v1.PodPending)}) for podName, pod := range pendingPods { - if numGPUUsingContainers(pod, rm.fullResourceName) == 0 { + if numGPUUsingContainers(pod, rm.fullResourceNames) == 0 { delete(pendingPods, podName) } } @@ -719,7 +718,7 @@ func (rm *resourceManager) findAllocationPodCandidates(pendingPods map[string]*v for _, cont := range podRes.Containers { for _, dev := range cont.Devices { - if dev.ResourceName == rm.fullResourceName { + if sslices.Contains(rm.fullResourceNames, dev.ResourceName) { numContainersAllocated++ break } @@ -729,7 +728,7 @@ func (rm *resourceManager) findAllocationPodCandidates(pendingPods map[string]*v key := getPodResourceKey(podRes) if pod, pending := pendingPods[key]; pending { - allocationTargetNum := numGPUUsingContainers(pod, rm.fullResourceName) + allocationTargetNum := numGPUUsingContainers(pod, rm.fullResourceNames) if numContainersAllocated < allocationTargetNum { candidate := podCandidate{ pod: pod, @@ -751,23 +750,10 @@ func (rm *resourceManager) SetDevInfos(deviceInfos DeviceInfoMap) { rm.deviceInfos = deviceInfos } -func (rm *resourceManager) SetTileCountPerCard(counts []uint64) { - if len(counts) == 0 { - return - } - - minCount := slices.Min(counts) - maxCount := slices.Max(counts) - - if minCount != maxCount { - klog.Warningf("Node's GPUs are heterogenous (min: %d, max: %d tiles)", minCount, maxCount) - - return - } - +func (rm *resourceManager) SetTileCountPerCard(count uint64) { rm.mutex.Lock() defer rm.mutex.Unlock() - rm.tileCountPerCard = maxCount + rm.tileCountPerCard = count } func (rm *resourceManager) createAllocateResponse(deviceIds []string, tileAffinityMask string) (*pluginapi.AllocateResponse, error) { @@ -818,13 +804,13 @@ func (rm *resourceManager) createAllocateResponse(deviceIds []string, tileAffini return &allocateResponse, nil } -func numGPUUsingContainers(pod *v1.Pod, fullResourceName string) int { +func numGPUUsingContainers(pod *v1.Pod, fullResourceNames []string) int { num := 0 for _, container := range pod.Spec.Containers { for reqName, quantity := range container.Resources.Requests { resourceName := reqName.String() - if resourceName == fullResourceName { + if sslices.Contains(fullResourceNames, resourceName) { value, _ := quantity.AsInt64() if value > 0 { num++ diff --git a/cmd/gpu_plugin/rm/gpu_plugin_resource_manager_test.go b/cmd/gpu_plugin/rm/gpu_plugin_resource_manager_test.go index ae8038da3..09a5c68b2 100644 --- a/cmd/gpu_plugin/rm/gpu_plugin_resource_manager_test.go +++ b/cmd/gpu_plugin/rm/gpu_plugin_resource_manager_test.go @@ -107,11 +107,11 @@ func newMockResourceManager(pods []v1.Pod) ResourceManager { prGetClientFunc: func(string, time.Duration, int) (podresourcesv1.PodResourcesListerClient, *grpc.ClientConn, error) { return &mockPodResources{pods: pods}, client, nil }, - skipID: "all", - fullResourceName: "gpu.intel.com/i915", - assignments: make(map[string]podAssignmentDetails), - retryTimeout: 1 * time.Millisecond, - useKubelet: false, + skipID: "all", + fullResourceNames: []string{"gpu.intel.com/i915", "gpu.intel.com/xe"}, + assignments: make(map[string]podAssignmentDetails), + retryTimeout: 1 * time.Millisecond, + useKubelet: false, } deviceInfoMap := NewDeviceInfoMap() @@ -150,7 +150,7 @@ type testCase struct { func TestNewResourceManager(t *testing.T) { // normal clientset is unavailable inside the unit tests - _, err := NewResourceManager("foo", "bar") + _, err := NewResourceManager("foo", []string{"bar"}) if err == nil { t.Errorf("unexpected success") @@ -419,7 +419,7 @@ func TestCreateFractionalResourceResponse(t *testing.T) { for _, tCase := range testCases { rm := newMockResourceManager(tCase.pods) - rm.SetTileCountPerCard([]uint64{1}) + rm.SetTileCountPerCard(uint64(1)) _, perr := rm.GetPreferredFractionalAllocation(&v1beta1.PreferredAllocationRequest{ ContainerRequests: tCase.prefContainerRequests, @@ -501,7 +501,7 @@ func TestCreateFractionalResourceResponseWithOneCardTwoTiles(t *testing.T) { } rm := newMockResourceManager(tCase.pods) - rm.SetTileCountPerCard([]uint64{2}) + rm.SetTileCountPerCard(uint64(2)) _, perr := rm.GetPreferredFractionalAllocation(&v1beta1.PreferredAllocationRequest{ ContainerRequests: tCase.prefContainerRequests, @@ -574,7 +574,7 @@ func TestCreateFractionalResourceResponseWithTwoCardsOneTile(t *testing.T) { } rm := newMockResourceManager(tCase.pods) - rm.SetTileCountPerCard([]uint64{5}) + rm.SetTileCountPerCard(uint64(5)) _, perr := rm.GetPreferredFractionalAllocation(&v1beta1.PreferredAllocationRequest{ ContainerRequests: tCase.prefContainerRequests, @@ -652,7 +652,7 @@ func TestCreateFractionalResourceResponseWithThreeCardsTwoTiles(t *testing.T) { } rm := newMockResourceManager(tCase.pods) - rm.SetTileCountPerCard([]uint64{5}) + rm.SetTileCountPerCard(uint64(5)) _, perr := rm.GetPreferredFractionalAllocation(&v1beta1.PreferredAllocationRequest{ ContainerRequests: tCase.prefContainerRequests, @@ -747,7 +747,7 @@ func TestCreateFractionalResourceResponseWithMultipleContainersTileEach(t *testi } rm := newMockResourceManager(tCase.pods) - rm.SetTileCountPerCard([]uint64{2}) + rm.SetTileCountPerCard(uint64(2)) _, perr := rm.GetPreferredFractionalAllocation(&v1beta1.PreferredAllocationRequest{ ContainerRequests: properPrefContainerRequests, diff --git a/cmd/internal/pluginutils/devicedriver.go b/cmd/internal/pluginutils/devicedriver.go new file mode 100644 index 000000000..0c7cda3fa --- /dev/null +++ b/cmd/internal/pluginutils/devicedriver.go @@ -0,0 +1,30 @@ +// Copyright 2024 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pluginutils + +import ( + "os" + "path/filepath" +) + +// Read driver for a device. +func ReadDeviceDriver(path string) (string, error) { + linkpath, err := os.Readlink(filepath.Join(path, "device/driver")) + if err != nil { + return "", err + } + + return filepath.Base(linkpath), nil +} diff --git a/cmd/internal/pluginutils/devicedriver_test.go b/cmd/internal/pluginutils/devicedriver_test.go new file mode 100644 index 000000000..b72a1bdbc --- /dev/null +++ b/cmd/internal/pluginutils/devicedriver_test.go @@ -0,0 +1,80 @@ +// Copyright 2024 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pluginutils + +import ( + "os" + "path/filepath" + "testing" +) + +func TestDeviceDriverSymlink(t *testing.T) { + root, err := os.MkdirTemp("", "test_devicedriver") + if err != nil { + t.Fatalf("can't create temporary directory: %+v", err) + } + + defer os.RemoveAll(root) + + err = os.Mkdir(filepath.Join(root, "i915"), 0777) + if err != nil { + t.Errorf("Failed to create required directory structure: %+v", err) + } + + err = os.Mkdir(filepath.Join(root, "device"), 0777) + if err != nil { + t.Errorf("Failed to create required directory structure: %+v", err) + } + + err = os.Symlink(filepath.Join(root, "i915"), filepath.Join(root, "device", "driver")) + if err != nil { + t.Errorf("Failed to create required directory structure: %+v", err) + } + + driver, err := ReadDeviceDriver(root) + + if err != nil { + t.Errorf("Got error when there shouldn't be any: %+v", err) + } + + if driver != "i915" { + t.Errorf("Got invalid driver: %s", driver) + } +} + +func TestDeviceDriverSymlinkError(t *testing.T) { + root, err := os.MkdirTemp("", "test_devicedriver") + if err != nil { + t.Fatalf("can't create temporary directory: %+v", err) + } + + defer os.RemoveAll(root) + + err = os.Mkdir(filepath.Join(root, "i915"), 0777) + if err != nil { + t.Errorf("Failed to create required directory structure: %+v", err) + } + + err = os.MkdirAll(filepath.Join(root, "device", "driver"), 0777) + if err != nil { + t.Errorf("Failed to create required directory structure: %+v", err) + } + + _, err = ReadDeviceDriver(root) + + if err == nil { + t.Errorf("Got no error when there should be one") + } +} diff --git a/test/e2e/gpu/gpu.go b/test/e2e/gpu/gpu.go index 52747673a..783d556cb 100644 --- a/test/e2e/gpu/gpu.go +++ b/test/e2e/gpu/gpu.go @@ -144,4 +144,57 @@ func describe() { ginkgo.It("does nothing", func() {}) }) }) + + ginkgo.Context("When GPU resources are available [Resource:xe]", func() { + ginkgo.BeforeEach(func(ctx context.Context) { + ginkgo.By("checking if the resource is allocatable") + if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, "gpu.intel.com/xe", 30*time.Second); err != nil { + framework.Failf("unable to wait for nodes to have positive allocatable resource: %v", err) + } + }) + ginkgo.It("checks availability of GPU resources [App:busybox]", func(ctx context.Context) { + ginkgo.By("submitting a pod requesting GPU resources") + podSpec := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "gpuplugin-tester"}, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Args: []string{"-c", "ls /dev/dri"}, + Name: containerName, + Image: imageutils.GetE2EImage(imageutils.BusyBox), + Command: []string{"/bin/sh"}, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{"gpu.intel.com/xe": resource.MustParse("1")}, + Limits: v1.ResourceList{"gpu.intel.com/xe": resource.MustParse("1")}, + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + }, + } + pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, podSpec, metav1.CreateOptions{}) + framework.ExpectNoError(err, "pod Create API error") + + ginkgo.By("waiting the pod to finish successfully") + e2epod.NewPodClient(f).WaitForSuccess(ctx, pod.ObjectMeta.Name, 60*time.Second) + + ginkgo.By("checking log output") + log, err := e2epod.GetPodLogs(ctx, f.ClientSet, f.Namespace.Name, pod.Name, containerName) + + if err != nil { + framework.Failf("unable to get log from pod: %v", err) + } + + if !strings.Contains(log, "card") || !strings.Contains(log, "renderD") { + framework.Logf("log output: %s", log) + framework.Failf("device mounts not found from log") + } + + framework.Logf("found card and renderD from the log") + }) + + ginkgo.When("there is no app to run [App:noapp]", func() { + ginkgo.It("does nothing", func() {}) + }) + }) } From a3d3e9e6878801d3729c711b25b9ac442cc5754b Mon Sep 17 00:00:00 2001 From: Tuomas Katila Date: Mon, 8 Jan 2024 15:06:19 +0200 Subject: [PATCH 4/6] nfd: gpu: allow i915 and xe drivers both compiled-in and as modules Signed-off-by: Tuomas Katila --- .../node-feature-rules.yaml | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/deployments/nfd/overlays/node-feature-rules/node-feature-rules.yaml b/deployments/nfd/overlays/node-feature-rules/node-feature-rules.yaml index 7e32d4c2e..1ccc85ab5 100644 --- a/deployments/nfd/overlays/node-feature-rules/node-feature-rules.yaml +++ b/deployments/nfd/overlays/node-feature-rules/node-feature-rules.yaml @@ -57,9 +57,23 @@ spec: matchExpressions: vendor: {op: In, value: ["8086"]} class: {op: In, value: ["0300", "0380"]} - - feature: kernel.loadedmodule - matchExpressions: - i915: {op: Exists} + matchAny: + - matchFeatures: + - feature: kernel.loadedmodule + matchExpressions: + i915: {op: Exists} + - matchFeatures: + - feature: kernel.enabledmodule + matchExpressions: + i915: {op: Exists} + - matchFeatures: + - feature: kernel.loadedmodule + matchExpressions: + xe: {op: Exists} + - matchFeatures: + - feature: kernel.enabledmodule + matchExpressions: + xe: {op: Exists} - name: "intel.iaa" labels: From 1de1024530f3d2f40bf2efa19f68708e0a978bfe Mon Sep 17 00:00:00 2001 From: Tuomas Katila Date: Fri, 16 Feb 2024 22:06:55 +0200 Subject: [PATCH 5/6] gpu: add xe notes Co-authored-by: Eero Tamminen Signed-off-by: Tuomas Katila --- README.md | 2 +- cmd/gpu_plugin/README.md | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8417a91ed..bc81fad78 100644 --- a/README.md +++ b/README.md @@ -229,7 +229,7 @@ The summary of resources available via plugins in this repository is given in th * [dsa-accel-config-demo-pod.yaml](demo/dsa-accel-config-demo-pod.yaml) * `fpga.intel.com` : custom, see [mappings](cmd/fpga_admissionwebhook/README.md#mappings) * [intelfpga-job.yaml](demo/intelfpga-job.yaml) - * `gpu.intel.com` : `i915` + * `gpu.intel.com` : `i915`, `i915_monitoring`, `xe` or `xe_monitoring` * [intelgpu-job.yaml](demo/intelgpu-job.yaml) * `iaa.intel.com` : `wq-user-[shared or dedicated]` * [iaa-accel-config-demo-pod.yaml](demo/iaa-accel-config-demo-pod.yaml) diff --git a/cmd/gpu_plugin/README.md b/cmd/gpu_plugin/README.md index 7019de164..c5b309f56 100644 --- a/cmd/gpu_plugin/README.md +++ b/cmd/gpu_plugin/README.md @@ -16,6 +16,7 @@ Table of Contents * [Running GPU plugin as non-root](#running-gpu-plugin-as-non-root) * [Labels created by GPU plugin](#labels-created-by-gpu-plugin) * [SR-IOV use with the plugin](#sr-iov-use-with-the-plugin) + * [KMD and UMD](#kmd-and-umd) * [Issues with media workloads on multi-GPU setups](#issues-with-media-workloads-on-multi-gpu-setups) * [Workaround for QSV and VA-API](#workaround-for-qsv-and-va-api) @@ -36,6 +37,18 @@ For example containers with Intel media driver (and components using that), can video transcoding operations, and containers with the Intel OpenCL / oneAPI Level Zero backend libraries can offload compute operations to GPU. +Intel GPU plugin may register four node resources to the Kubernetes cluster: +| Resource | Description | +|:---- |:-------- | +| gpu.intel.com/i915 | GPU instance running legacy `i915` KMD | +| gpu.intel.com/i915_monitoring | Monitoring resource for the legacy `i915` KMD devices | +| gpu.intel.com/xe | GPU instance running new `xe` KMD | +| gpu.intel.com/xe_monitoring | Monitoring resource for the new `xe` KMD devices | + +While GPU plugin basic operations support nodes having both (`i915` and `xe`) KMDs on the same node, its resource management (=GAS) does not, for that node needs to have only one of the KMDs present. + +For workloads on different KMDs, see [KMD and UMD](#kmd-and-umd). + ## Modes and Configuration Options | Flag | Argument | Default | Meaning | @@ -205,6 +218,31 @@ GPU plugin does __not__ setup SR-IOV. It has to be configured by the cluster adm GPU plugin does however support provisioning Virtual Functions (VFs) to containers for a SR-IOV enabled GPU. When the plugin detects a GPU with SR-IOV VFs configured, it will only provision the VFs and leaves the PF device on the host. +### KMD and UMD + +There are 3 different Kernel Mode Drivers (KMD) available: `i915 upstream`, `i915 backport` and `xe`: +* `i915 upstream` is a vanilla driver that comes from the upstream kernel and is included in the common Linux distributions, like Ubuntu. +* `i915 backport` is an [out-of-tree driver](https://github.com/intel-gpu/intel-gpu-i915-backports/) for older enterprise / LTS kernel versions, having better support for new HW before upstream kernel does. API it provides to user-space can differ from the eventual upstream version. +* `xe` is a new KMD that is intended to support future GPUs. While it has [experimental support for latest current GPUs](https://docs.kernel.org/gpu/rfc/xe.html) (starting from Tigerlake), it will not support them officially. + +For optimal performance, the KMD should be paired with the same UMD variant. When creating a workload container, depending on the target hardware, the UMD packages should be selected approriately. + +| KMD | UMD packages | Support notes | +|:---- |:-------- |:------- | +| `i915 upstream` | Distro Repository | For Integrated GPUs. Newer Linux kernels will introduce support for Arc, Flex or Max series. | +| `i915 backport` | [Intel Repository](https://dgpu-docs.intel.com/driver/installation.html#install-steps) | Best for Arc, Flex and Max series. Untested for Integrated GPUs. | +| `xe` | Source code only | Experimental support for Arc, Flex and Max series. | + +> *NOTE*: Xe UMD is in active development and should be considered as experimental. + +Creating a workload that would support all the different KMDs is not currently possible. Below is a table that clarifies how each domain supports different KMDs. + +| Domain | i915 upstream | i915 backport | xe | Notes | +|:---- |:-------- |:------- |:------- |:------- | +| Compute | Default | [NEO_ENABLE_i915_PRELIM_DETECTION](https://github.com/intel/compute-runtime/blob/3341de7a0d5fddd2ea5f505b5d2ef5c13faa0681/CMakeLists.txt#L496-L502) | [NEO_ENABLE_XE_DRM_DETECTION](https://github.com/intel/compute-runtime/blob/3341de7a0d5fddd2ea5f505b5d2ef5c13faa0681/CMakeLists.txt#L504-L510) | All three KMDs can be supported at the same time. | +| Media | Default | [ENABLE_PRODUCTION_KMD](https://github.com/intel/media-driver/blob/a66b076e83876fbfa9c9ab633ad9c5517f8d74fd/CMakeLists.txt#L58) | [ENABLE_XE_KMD](https://github.com/intel/media-driver/blob/a66b076e83876fbfa9c9ab633ad9c5517f8d74fd/media_driver/cmake/linux/media_feature_flags_linux.cmake#L187-L190) | Xe with upstream or backport i915, not all three. | +| Graphics | Default | Unknown | [intel-xe-kmd](https://gitlab.freedesktop.org/mesa/mesa/-/blob/e9169881dbd1f72eab65a68c2b8e7643f74489b7/meson_options.txt#L708) | i915 and xe KMDs can be supported at the same time. | + ### Issues with media workloads on multi-GPU setups OneVPL media API, 3D and compute APIs provide device discovery From 4946b26018dc600eef9e912522684ac8e4355305 Mon Sep 17 00:00:00 2001 From: Tuomas Katila Date: Mon, 19 Feb 2024 11:01:20 +0200 Subject: [PATCH 6/6] gpu: doc: monitoring resource notes Also align xelink-sidecar deployment with the new files in the xpu manager project. Signed-off-by: Tuomas Katila --- cmd/gpu_plugin/README.md | 2 +- cmd/gpu_plugin/monitoring.md | 32 ++++++++++++++++++ cmd/gpu_plugin/monitoring.png | Bin 0 -> 49702 bytes .../kustom/kustom_xpumanager.yaml | 5 --- .../xpumanager_sidecar/kustomization.yaml | 2 +- 5 files changed, 34 insertions(+), 7 deletions(-) create mode 100644 cmd/gpu_plugin/monitoring.md create mode 100644 cmd/gpu_plugin/monitoring.png diff --git a/cmd/gpu_plugin/README.md b/cmd/gpu_plugin/README.md index c5b309f56..e706bb362 100644 --- a/cmd/gpu_plugin/README.md +++ b/cmd/gpu_plugin/README.md @@ -53,7 +53,7 @@ For workloads on different KMDs, see [KMD and UMD](#kmd-and-umd). | Flag | Argument | Default | Meaning | |:---- |:-------- |:------- |:------- | -| -enable-monitoring | - | disabled | Enable 'i915_monitoring' resource that provides access to all Intel GPU devices on the node | +| -enable-monitoring | - | disabled | Enable '*_monitoring' resource that provides access to all Intel GPU devices on the node, [see use](./monitoring.md) | | -resource-manager | - | disabled | Enable fractional resource management, [see use](./fractional.md) | | -shared-dev-num | int | 1 | Number of containers that can share the same GPU device | | -allocation-policy | string | none | 3 possible values: balanced, packed, none. For shared-dev-num > 1: _balanced_ mode spreads workloads among GPU devices, _packed_ mode fills one GPU fully before moving to next, and _none_ selects first available device from kubelet. Default is _none_. Allocation policy does not have an effect when resource manager is enabled. | diff --git a/cmd/gpu_plugin/monitoring.md b/cmd/gpu_plugin/monitoring.md new file mode 100644 index 000000000..3b3050aeb --- /dev/null +++ b/cmd/gpu_plugin/monitoring.md @@ -0,0 +1,32 @@ +# Monitoring GPUs + +## i915_monitoring resource + +GPU plugin can be configured to register a monitoring resource for the nodes that have Intel GPUs on them. `gpu.intel.com/i915_monitoring` (or `gpu.intel.com/xe_monitoring`) is a singular resource on the nodes. A container requesting it, will get access to _all_ the Intel GPUs (`i915` or `xe` KMD device files) on the node. The idea behind this resource is to allow the container to _monitor_ the GPUs. A container requesting the `i915_monitoring` resource would typically export data to some metrics consumer. An example for such a consumer is [Prometheus](https://prometheus.io/). + +
+ +
Monitoring Pod listening to all GPUs while one Pod is using a GPU.
+
+ +For the monitoring applications, there are two possibilities: [Intel XPU Manager](https://github.com/intel/xpumanager/) and [collectd](https://github.com/collectd/collectd/tree/collectd-6.0). Intel XPU Manager is readily available as a container and with a deployment yaml. collectd has Intel GPU support in its 6.0 branch, but there are no public containers available for it. + +To deploy XPU Manager to a cluster, one has to run the following kubectl: +``` +$ kubectl apply -k https://github.com/intel/xpumanager/deployment/kubernetes/daemonset/base +``` + +This will deploy an XPU Manager daemonset to run on all the nodes having the `i915_monitoring` resource. + +## Prometheus integration with XPU Manager + +For deploying Prometheus to a cluster, see [this page](https://prometheus-operator.dev/docs/user-guides/getting-started/). One can also use Prometheus' [helm chart](https://github.com/prometheus-community/helm-charts). + +Prometheus requires additional Kubernetes configuration so it can fetch GPU metrics. The following steps will add a Kubernetes Service and a ServiceMonitor components. The components instruct Prometheus how and where from to retrieve the metrics. + +``` +$ kubectl apply -f https://raw.githubusercontent.com/intel/xpumanager/master/deployment/kubernetes/monitoring/service-intel-xpum.yaml +$ kubectl apply -f https://raw.githubusercontent.com/intel/xpumanager/master/deployment/kubernetes/monitoring/servicemonitor-intel-xpum.yaml +``` + +With those components in place, one can query Intel GPU metrics from Prometheus with `xpum_` prefix. diff --git a/cmd/gpu_plugin/monitoring.png b/cmd/gpu_plugin/monitoring.png new file mode 100644 index 0000000000000000000000000000000000000000..c56fc5057f883865a3c2cb85adab1ea811ceda80 GIT binary patch literal 49702 zcmc$`WmH^2vo<<065N7o26qb%A%VdyxHG}s-ARxP9&~UGZo%C(XmEFTce|VSoO5k` zKfZPU+`Sg;VNdVw>gww1dP)fTDkqMLM2G|eflwtSAc`Q+D}E5@CFtEt;0gWDjZEN% zXe*)N00N@4`1Fk}&MU8B7VpO!lV=J+WL!Ul zp6{Sf2tla8t##@{*$d!)%}2@g{22PP69Kp_@BNy7{?n5L3m_ikTZ(=9Hn222JbZU| zcXQMDXUB>y&+P0hF`BrCN5hrP-0UnLAK!K9YoO_#X(tR59N9ZOEboYU^-W!UdV3`6 zhJ|R-n=RDk-@{1{Sz%$Jg}HeF@cTd4jW`A-=21jD40D?wpmAP;goFg`=UQ%>xBZn8 zG1}XwcbVzw=_hMFJ)NDj?~&@XnIk2x#Zg!~=*z0At5f-$`bdV%&CQ#i9zCvVA>g3k z;9!8z@NiO2^B9*;XJ==7ZfL+9j(K&sJH2mrXnuC+aV3R`ttD`n3>a73`R3*2*{yZ| z-8kLeHU&Dz3UW{V)=PvIG`L|O3x}&IE62d$PuFV^5XU*I8d2a96YJZsuhS;Q#qIWa z<{^p~V}I(N;YY%~y}ck>;9`5e>_&e}c2L)&9(3kRm^08nU%PvIM(i5;`m3Em*c@=1dg6Cq8_248q|c&%Y&Mw$1R~JUYEo96qiL_w^PH{oWNW7 zzX}TIuRIGJwuS~%1^Jnnn7Fv&e14S@z6>g6C`3$;48xHFkn3=-|cGG-*rsrpLn4=Fz>2Xc~Drd3kvW ziDj~9@ny}cx1U@88;!@LNeS=25uZ10j-5U|J#D_-EdS|`qN?HmEKWKA?CsmnK)#T{d7#@o*@%#6kLeINpnXp2$*v-xDCmP9Vm2r|h zC6|i@6Cbe>< zKCbLd@JtHL!A!+g#qcL&G&FCZD;0X27SAx9sm`3Ky{S?t%X_3WpGP-=ZlEbBFfKDQ zGeYR`dMnMR`7tba4?Z1E!hd<#a1`k8-*`SMU9;d!^j{R5$pLE+8#^&NdiqZUeZlAH z;r^7LWV{u-|9E=h@X(*NAEi2tDR-5S}GIuz=2)O7Xf{9Q$n?oiD8?`>bTweN4X(iq+ctLeH0Kwp7;Gf?bMt>0ad z@rHDrE_xhkFyeaOh}{9brGKS;JcYO5qab)&{X+59A2Y8a^S;&tn0b4#cLR`%--j`^ zjBtAL?{ZX{<;~l-rCK#5)zxuOA{4*mX&LGyN&FaOA7c7m( zFEPE2+uq8`%HE%<5Dj!1SIm@GRHVplDai{`#Y){60sWU_Otp$%+na`jhI&5UYyk&@ z(Cs$G&yF@0XiA&5+wtSwQ5v8Iu3DZ1QwfkjtjXjh-{ns>0^YGFFj7+fPETLHzc~Xo z#TLA@J)No=*hVrg@ulr214J0 z@MnQOOn^R==wlK5GJgLy1T?!Jr@SS`W?5BL6+i?NHMKFIE$(MqRct(V%9cKz_S%rS zt=B|6Jlg!Uymz;e;;x6DP6vt<-xGpB+8;VOkL{WTa;<=Oh!3Wiih5BBnQcUYsA`xers?*Zv~T=wEbGH$QWGs z%jdkW{^iSy7H82CuCmy;xWmP!JMNa7Fsba-DTMC_|Ae^7j>L@t;rQZTiy#6U4y<-N z+muZexantU`IsOBP_Cn*e+Qh}HCA(6dhPxL12WzRyNEDYp4n3li(G~i0?2mZdAPNT z3?>c^2jKAcv$r`fzyeg_T#@Qtx(u{o+JDAD_zeVt8o;YHOkVKT=l&QM=$(6*!!oosFw*v^{))ayye-GuSKl|aQs6Q<`6P%5X8rQ;Z)NV)S z{Yg&(EI4X8!BEQ!Bc6(vN71w!{~7#bN0 zR0{!%!{;n%=@{AX=Nb=EV!7i?3;1Kzckxom z?;c>MBOxQ33?^oOrM<5D(piNa>DV1?^j6-8%!aCG=isTqW;ev;=V`=Vbn(Z#;kw-U zd|Dp6wkHjC8~e&F=igZ?=f7woIjGjX?dOFHLz{A9(NGe~i=|5Dh z?%{5V_WJ!DCPALUYO^!yaR_s^$=Ldn@~Pb5tmQJr zntG_xrsHjfO(PHYOBab3%iUkJ{q(xu@>@A<4$L&T96dhVVPms+_?5V*jT$HwPT(Aoncf*&`l-fH(?pvBtg!9xk?Ls` zYvLtRXuUU?%2cu2#4iw5KpFqAAz`+v821R~m9wIU_10&-)-mOV%(7`R&v;df0bs&}l z0;wLAeOOe4^E=(g+w8qk3G4zRv}L#&e%f3aS8E{;j-x@wP^ zoc!D6{tRK*X2VhQ+SIMyx1hm=Z%Kez4H(@wRTfehO~)e*3(-xMR7usl%$n{qL?LCf z(FbB(-BRw8jjuyP5FKb}N)ENBRuQDDx0*+> zq5{=&<9kEf<#7l48S)ePHE0Cw~S8x(}DOg9Q?b)H3uk zAh=-r3GR?ZC!+q)fKZEcT%@mn5m_s-AHzgwH|`vT_YpRS%B`6R9|Xai|HB6>-@(&Nx0 zb!NTS%#BOq$>;OzJ2l97U$wiYU8uzTROF++@$1MC7|cj3=p_thpdnv1JjMR#eg6Sm zop8~}=Hfo}GuH#xV`?_^89uV!4+7w7so5(aAmEj(<*Ql-cM;=17Z|^4_n+oJP=Qw& z8!B8TPWLhKyRz89w0{^SBHYG$IG>LYu8oMd*X@Y;7kB|ct1T{sWl$VEk`bvS+oWknJ`&8E&-xTdDHWp?fy)Ln$2Q=kV*iU+3uyzB3dAng1 zulX~7MqnsV1TdSMyE}AmeSU|{&>73-MUOuhUohCNY8)0AM6JZ_9WK_V_AIMUlyVTT zF(do!w_A4>4d8{hU+t>y;G>;%Wjb2~IQ(tVi<Ne8EMo~`Tb z^Q*a)`C*+IGz8#}u0-r6O{UYk8#Y`#K&#q}z=W|zdKv68K& zvOXi00xVwGWoz2Ar0edK#P7s5a;7?TaAWDW?SV&KZ)0sAr(@GnuQ&IoBql8);<^j8)^F*5nm09Mtr>Z=FPCX^zpc zrT}AgF}j(esRlp0T$H1tqO1Y^6da7?y@JKu(jYD?8#ZI9`wVbwJA$Yg-PXgie0ayF zWZ0)qFIHi_6N`t79#einy0x2iX^V@0&Kw!+xE`8`ADWX7&2xR@-?aFo z!rueS_-z{b-8%^>DL(J}+Kv^BAm$d6&J7Ps@d#2XvIeSIDsr=(U^Dohyd*Zf`4SF? z4<=yrMHo`Pd#AF{Rn^A)lx*blwyw~$tbyDJJjFQN&sh~+bu50(`Vs{qK|jJPCRGHdmdX}psiGAo!v#$iYqB%5vvLOqI?^%-1N~@h_!75`$yJI)&otH zSSM9Ip49XH!-)J$9{~bo(4Q@!otFa>;x^(Z-LZk`1o$)w5$pI zNwolA=jiy+=l%2Zl35rg1(<@Ol&g$4MXq9o5%9+RXB>Q#$jC@F54G`>>;G~smPE1{ zbpK(r;7Ym)NWGP^RCj)U-&QsZK)if&hF;qYWfSA$uMrURXP!Mgt(>d6thOpa#r5@s z{m=X)Y%M{=eo^(xhi6B{2Y@*YTd4e>(<_>S@dB z%S&hPwtgyXqW*FSC2q)nQz!8K`*+Cr=x95OYn`FKLkr4(cV*vGJUOZQpBIuEdN$F# zp1ZOQd;R7Orn}|y=jH*DniAGBJ4t=BKVEHR+_ z-+z~5?9Xfi4%Oz)oTp{G4%|$Vx;6yYY2JrW zt=I(<>_}zrM+t4-W!06NL%pTN?RP&4l%6Icqo6dNk`hq^eqvvi$iUDXF{K%^l;W-0 zgZ(o0!WnfXP2(etZ}OyEi34&8t5U18zy7t&?hS-7d4Cu|O1Nub)6*B4v}d&$)*ALM z6P@bzd2yZBPhBk?XcK1?33!=Xb9Tj{A)X;KBjP5mbjvg{hN+Um@ z|3j^ltwgSr_g+V&BZ}2ibr$=~dgvg^GvXf9jl3I`v_l2M`;8PwN4r{ z(hH^M=Qp+1-9z*uF{6*96LDt}?u%}R&&+=qXAh>I4%ehu#ZVQyEAG&_+2L2Yy>TLm zY|-XvfxoqJP+hBea~mx-7jfX&->YB{?c+9O>Bc(d<=byeKYOuY*MZIz_jId%Eb&!ChuliS6@o5A1UvmckJNx*>-LFWpa=cFqfjqwVEq zs0X1R%?AcZ3ni=S7un^rdQT%NR%jfXn<|fd$^k00btM9bTUV&t;Dn z6!Q)`q1oZ4WH6Ci56Hwa`c@?uIXaVAjwd8a7DZEYkf-Ca4tu^l4^YhQ7oiwVBiWkfCTigGFVM!@4Ip2=9K>3_QI4FLxK?(r|vZjY+#pBk$IkM5f?{_(W$Jg7~Ysa zpgDP6{gi|-Z55yuT+j>(BCNQ7!F%@LVHk(fx3&D~bsUu_-~of-BY8DQI0=}euHqzLgn)i#}@~H{^9{M|| zlcsiG$9EZkf*7fk2K8*GmUHu-!5&)>q&G1i%H2Edcd^JRN!*a{jJx04Cg7r6E6(m; zofk@tMr&-kA)AkNQX?F!V%Gnip$B=W^@Oi%UxGb8pEj|SO4`;;;u0cF6?4|*dHhhe zRqUfs{7FfQT1~VX{P?VA=E;SP$_0E0FY3jvUj99D;zX5+m!hIByT5eRi!SM2|MX8Y ztCULN9zQl7HjVu=Y*6NU|0cyP0QS4@{5_%uY7c!wU>>JBTMI|YMeC%MTVZ;hXqn|; z6NiTu&kj3&c)VmHJ5{mALZ0otwu-=7(Ri{I7>q@)#`v@+-XG_-N{ETrQvm*vci>q( zC<-as@voLm|2lNi$q9}csAXZUJ>!1WI@9kk(SSOX?hloVR`6k7@+|6>dMt1>no9Yq zFh9<}=5g-y@_m`v4x(hF>E%4e_4EtSHwi?kubr$aw+7ug1HZnOr;TzsLu3{DU=r*f zKKNHBo^L9PQqDM1W7C96lHx%i=OV>3bQlqFS14L53Zs~Y`5YC$PaUJ~DJa@Kh=n2J zZ5mhB)6M+??tkmucHq4XgH>U&l%Wd#jzcAYTir~;m37A=i4oEUc_~Ir@4$sG>z=jH zU2O*;*(93{;<3k!-23q$ATFccOIwP~s?e52u*wwH1UoygW=W;nezbDHp6iSJXx-PB zi}fSu*6!wUYmZ3hJsA0{na|Gkw|aVq;8(JdJF;hk0!kHIxEer1FH2*kG>RY6JW7!K zse56%UY65X*}6-Tgr7yx^q&T?%=CfeA?>EQ)dxbl+O9hR2)L}px3`Ti?`$zmg0Xgc zGg>;WVl6$^DM)-I+a=bM%)7bF3T0J3CH$=K*fdn)?_e;A^j{3w+2Zs}#XG_&)v_3z z%p+qeSga_R;s?4WAHXFGJ7E3z!?b|Iv-5RJ^XNx+qoPkWj(35p@in8X?saBTp=2Tu zW_v6*aYj6Gh_`Uyhs^o|qH!u!jqb!Z=#(DTjrESZxlp<*;{JC8U^S0n|MA29a7Q>L zaZ1#!`tX9Fd;R?eGCtt>+Dft`UQIlz-A=;-Sp7*ZjuNggw<8szU}9zjj?oz_j(=y= z=#!MOSu|olD^^a~WtIBlqcXK4CnuLmjqwdq0uNxAt4Bvpk)ZU>|GL!5byrsSq+Nws&w8~U1VX)a0W-wUKg(!maV4&2_ zmCz1SK7vGG*G*VT!^bmh8tQkVJlqfUt`zRZ(8@2~3peHJhzy}FtDAQ?Tt@h;{~SJ` zvDA;M?MN3grF8o-@4=a^Oa+g=NS_`300?wL&jzxhao?=my5*9#vPt!9v6w+yV9Fb@ zmFv(Lf+ov}5r&h>*sC1zYL)f<8Ml*EL;a1$H!F{u!4XSLOx&|XMPav@?M1RGgNhsN zZ&qG{zl_J@6L7*~EqN`AE4T>J;KnVP*72X{!ZUKD#&|7~{MwaFeuhjJ6d&P|jcdq? zDM=oNx9r_FT}?IV!<+8YV}6^lvv1)UZV?UR7YCM~wzL4clWn-^*CNvHqJ zG!;Ze_T?}^D=nKXK|vCX(9`jlF$>N^M5{(w$Bl%Lr&iAjH}Ml_8WSZL?4xlS^Q4iw zE9oC9aQ0VU-A3boasguMU8Ip0A1i$&$m6mZQ?6VC^BSA`l$GyykIYpKg?^>FJ@w_3 zau;%vHD`XvBo}QjpOEanaC2I(pB9%-3-2RK<&HTmcZH4bpR&?>h4__fq zM!>g-3nZ5`vDmKA+WTu+m%6u#GeJvJ5}%lEN4r;&cuFkPX9{m^RlRWHi%h!A=F*%L ztkAx$--S!-z*!ETHd?}i0bLd~@xj0-Yx36lcb(;-a0GBVTT{O*<4LXw>LSOMGWq~- z9@!-$-Wi^I57%?wU;gl|b0u_5aW&C&@W-Jt^c3n5y7kMW0WyOS7li~>kF3pu| z`}Kpw5jbmV&Bl+I1+iG_Xhaq-w7<7aSboeqBQhyaG=ZE-b4MK%xm$?djZy~An)lOR zn&Yg&sc8iwn7>`zbUmr26@RQCUSm-TO{(~4RU$Ruk@M1ob93uGJW+Hk&HbBRvP+j6 znEwijCy1YVPnj=oJfw8|^UioIc5~C*f~U);DVEFbT0mn(^LawPD`F=Sm(l2Kk8tl} zP}+yBHJ5?>qf!OkEPofTN^=W2ho}Y>EO27o?MYBFYaT}l;5oe!;bc&uy5DU6<9peC z`XiXBRBJB{dZ}(dwT7E)eDN6J&?n1SCowv?Int%7|cl&okmWGku`upoFk+2FylNCcd`Zs{r9oIz=Ib@7v zA+tIp>d4DYR+ttOGkZAv`Z$0@l$RnlAXm?bPl#kKX7r3sy-v_oF?xttu|L2tHN~g>UD3#W_kJe|lWVNG7>6d5N$BQH_dp__ zkl_3tL3&&?9~ZfG=nq(L{;(QZW#B>`*i1mAN^=dJGBq~Cu?GMKX8T?{?6TGqk$sgs zIGC8yWII`U+4~&cesDMndtax6>h9)`sl`Eje;=WgZjvgxW)c zNz;tD{S8E1o=~+O7LczEzx{eI5Ka&K$EEy-iipcc*$_QgNiGDhVW2*Y1%FJD`0a#q z8>5SUMH=%cslvNAZ&7+4*6Bt4JGC7cp z{n^mZQ>XW1%?~24|J)>;0vy$L_-aL=q18N#RYZuLWnRd{u|LSBS}kyWJ8ElAqZX0{ z_{G94T##mUAdn)n{ylzY0{>GDJM6UwlByhoI)1eGvTp7=i3Oztqg}e?DpV1CrN+Z#L%gJ2tLS%nxlrh?uh4d^VXb3Bx0^!W5bJ9s=df zH1I*qg2`DEI?|zECl;hDExIH_WPXqJlGsW7B{P~n^;Sbo?X6lseEe7i$SbIhiEX@E z^mu0_;@5Bcv9y=;SMp;Ir(^~MGUS2ed~c>iBCg}G81ND-mn#@sX8Is?CH!Km$qm|= zSomZ<74uh`Wulnioe;nR<(eYbgUFdf#&OQ6rQni(G>f`*s~8B_oQR zhYcJEtMgq5{G-#fFAeJn~cyTnj!Ac56`4fS4rO7QlNJt2K7cZZatMu z6)a5*xEjy&k99%TrQk&#IoIne6@xp7htc=x920Dq`31V{u@a0}zx<4D)?V6kX;?Aa zLk-eut5xHAO9y{5FM8Jw*?&JHqKlgv?_c95DiBcjkgxUySJ)R2;@d$C$u))#`-ikDs@0cpUT3dxjA9!Xx|hiKesLJP zn8ug_1@g(8isV|++H#H0>By)z#v+xWDZWI={8Ar6 z9STD_*?oXPydmq3RoFW7Y!zIS z0G^6)Dfm_WjKpVq#l+0rr8geso%v@b;;*Z?cafx=2_=Nb+k`F^DeZP7+r0JiL$JF& znm=DcoEHOM4(3>`Evhf>HJRaSbF1k;uQw^iY#YHO-A zzbH6RTUIW7k4ZrX^Ggf-NBt~GF|l3XXwgOXJz zYb(&Fxyw8krFd;jSA%2BMEO7dMD5;KQ7iL(f4f61WeKwYbs!4Q{UZmQGi$T%DE9NB zL}m6vse`1?xTqZy?`Pazt2`|jOE&FlQEBb^MX!9T_M*Eq&;%TF0{5)@jN4$Tvu{&0$ww4q_#sm9gNv&P7a~sNVsXz4ePOZwUX^|> zblpFJ(hVA$Wgkjjpa{i}ZQk8eQ9QgMo-;ioHM9tGUR=))r@ni0L}#QOa_|ySEI1pI z5=LdxaPu%cpOjhs}iRaIh3n z_8lAiVp;`Cj;(5ZbbmFPAKeqozdO8XVnKvIQ##he3W2o8`@b8}8(U1)pIM!L*R7C6 zw_t|R-({Ft^2_A@-i7~Tn5=Nf3%-!sSlIeAD z+F2a2AYzb_M~WM$iGq~o5C0<00Jm%!Ispf_EkfNP&8Q=d1PhKz4BlZ}92@R;K`?^C zWH=+DM5?{Kwo2;l9`{`CA_s%Obm?Hh#L-Zxc^ti}A^p74igsZ8)17d>*W{1=iLxdj z`&IRT(Rv*QBkp#mv70o|;&gh1K+1QNjL+3mU*ZvMrlA9z9|UC#{tUwDPi=sgb0*HG zh)duT<9SR)Ct#zx;TQ>e--rI97AQ|Qy6b^jmhlF`A_IZbHgcw}M%2{(w!DwziGI{D z)(qyKQ9u{*iFb1vSqxi&f8=wjm=oeaJU^a4e2egGk&8WeeCja~93ky>=rppqp9O7d zatpb`$hVw5746@=vZ-LR+c&V-d}D-%XM99U3C5Db>C2X1jNR1Tr3=?Nnf>7_kA!YH zt^FCIOVg6s5Hci{5)d7(ba}z!JX4yMv^A}sH#Db|3Mij^@MXq?ixDkf3-W>L;N+@( zDeeYA)g}rirsS9r!C$Nw6k4^HUlcBD8aTViaO==d(B)H4h5D6Dw2e0`x#2Ul9BO-2 z_tj;TrSUMBe{@IGuW9BjJfWw14xdmMY_b$853!gmTEPoM>*VxJ>XTne@}Xr2g26tU z?$P`|vF5Rz+*=AqZUYovvL`wy&sAx}r*v|#!DV@4RBvU?QX-hbBzk61Dsx%lq>DGy z-~bgb`8{oU408fJ^bNfh_spmcbc?RZ@(fG6UMjy^8x$=&a-&OwVc>B%EZUXIO?_9Hi4;CMz3`_SE`~ z&PI4TexQalrRR0{F55ehK*w$g?J6OtcI7f}7?XANs|GBTY^qn#g-P4_M&k^`U9%GX zYzedy2@`=(XgVmLJ8td5m<8gm_yDT5Lj^Iv$v_426F#Ipw@bHqX9Z$KK{@CIvQI$| z-~S4gQo>lTcpXaZBqfY$it5*TfUrl|LwFHuJ?&|qEIe~ogl7GfEbI%#2WL(PF zNkjl$DB~#qxG1AsEdri{Mn^yttgdkKbXjQuTVQOlqU2v?DGhulRjrVKR!O*v(Fd( znqGD{9m9(ut_U2^O(oxg2jV1^IGQAgM5gjlL_uF3OG;%&3BBf4R#F-RGNbMLnQ8>C z<{kbk@0y#9lIz$uwfFCXTEduz#PSvkDV+)>!;y?yFaA;AXI^to91jMAwK(eD(p`b( zlCE^Mt(pNjG5uG*49kvu73JkX&B9cPPDm|+fQ040zjhfTmbeh=h!9*4F}~TEv#kQ8 zzVqD$MEOSVBSW)@wePUn4t}4j@5e!dS8{+ib#7DWTDN`-o$NhGJLqc3rv=EW+{sM~ zkBHcsTFwD#O$1+4KTI{geZNRyb(?JEGdSg4FPWiDLbR;&B6GH4QZ;z{w%GRsF2%O@ z8dTCM^mr06Iy#EOpemEX7Ylv$S>Lb!RE;@>PoqgBtC&~1jQ$FL&Q(Uo4l<>_)Q))n zG7C#zi6dr>pWgwpUOs_yNXt@MSw=}nLgq2N+H6wFi+Z!WifnMGMbF=qf9xBHrDpiZ z7Y^$+DrA_=7PP)@o0O{wB_@GqN~KXDpSl|rDm*zgFMc!9@s6^ii}p@o9&K!R>Xex1 zi|zZC8Q9TvVF`ND|tN@#kPz)ab-?>>ega>3_>c(cH(;Uadk8m)|y*=SU-!oSBs z>f(EgC%Fp$-%agu)Nx=-vVHR$)4l38cJ+KwpuA_tTLdT3<_v zM};_5+Cw0Q9M}6Hp zJ)E$zraebIhm@%upbN!d4 z@f-G`Jh!{pl!m@(@&GWK+^FZb;iMV9rXgiB=UO1k@}j6rLdU4_-_1)f*yMoCOHV<^ zVC$%OH>I@sk3(*WmGXTpL`6c}=tfAl*R3>t#0#O^yh7FHQT%%l5dkhX*#{sp17td0 zwWr$0!Sei^{#ZfQ*Jo3-8!G(rHnbFljFuiDJ2 z;zyqqI2t10v}yihAgw_0w|$*G_Yn$hOvn<#fYDSt@{6M*egdzolI-pOkN&0`2w7Ko}bDU|TAIYWSBe@tkTKsvmP1HkY5Z9Zm0 z00nz-OiDM{m`q$=mwj;a>N}|7OAYPK=nu&#s9e4)yFmRDFE6iBj?B%0jZc5+3)e3Z zsJERnGW?(`Wvria_*!oGRz6v%<3DM+3r*x)zQuGS(?Pu>tK?vLAb0|Msk~!j>3;wB{8MR9QrdmV zo77q{$f-Y!+@4EQOB>Re=%`7P^>uM4+yX)w$C6yS&4w;V_=!SLuiE@G{0&H?%Vo(6 z9vh2;hgV}hK@Ug<&G~ftTaT4JGP0de6cQ`6q~b6NAGg~^n$4f@V7;^IQ*tuDi?%j# zKCDCvv(N6C2fgRAXi_3aL%h;up^FRXhnY8tF?ahzJd9#hvnHh84^2B6iIrYP^+RQ> zHoMOtr^uIQ#B6#W78}$#6xuLRRNTYPizi)EDI$S10NOoLN1b$X`$+;K9ZH4Yv=lBD zNK22EW^wlR<=O;fq_anPo7DIOSceZ-Lf?fjd+Gp{Se3Q~3qnD>D0oX1O3i+)Cy6qpxy1o`n4C&sTlpG6s89|7>>Vd>lOV##@X z+$z9@|L(gCR`;)d^CQhak`TxxLSYF0UsD3ipzOAF^YXbyhqop|HU}pk1Nx&NQl0CT zjsG@TF44Dd(ty=~z1BY|MzO=wOl;EKDCgpGZ7CJPJYm;P8l1d{eJ9dE`O@6X9P@Y) z{CR(P4~*3Lp>IbrLH^Tm_a2>nNtwEp<9Qww@rvqCqcrQJ7RC%Hur7>9hk zYrcR7^$TrWEFu(`y0=H;FAhSfCN)vbVO3>5f>gs!WUkVa8=fJ}Vc9#f0h(-&Msn z&39qDhL|EFcRv?Ty|6iM+{1}70Fph&CB?F)5>gPt$}A!F7Xs>A?-)z&XyfG&uf_^W=*zV{B zcq{3=)pgq*2G~Bb!UPWu6jsd*W^wPuf+38C@2b2obo z6wKU+rVb3!+#B|eRw~N$oM=pD)w!chu%%ck~*6e82vLK`bfz%Df`T zV2dR(t(W=f*y`ygE<=xG5N8saB?I$Eghj#r2cf#-@pD_JaJ#*RfFE6bv}IYUN{OY| zi``$^ow$u?)h4Dy?0>N%Ai)HW9jwyad|)iE8<=iEr_blOTy@12GU`AoGBG114@ zjxPz$T-t>@_s^+l!SM6%Q*+tZMblEtMtwCERtS(&4UmL}86LLZgiR4;GwmmT#^rr_ zjGY8Rozg;{*)gym?%T(>n@%)RIPNOqL9qq=>YjbdCm@$8uKrrY)vnthYW1=6z2^GAWGI;0$xoO({j6Dvx34nc&v@zwIVP$u$EN zbwNIg-IR6Jm6ZIY{t89CL&V$BppGpT?TORD7Np9gMP@vhh_$@Dn{gSdkEjb^2kRL= zJw9kvTPAQ?Op2fjc9y=-9{Nu`^&>teW-+uS6*gI<(ZO;Z*x1-u+I&lIzKG@hq9jQ* zsVTqf?`tVB$$+LRpV`C#reaD$96D$Dl@`Sz@ZKgnUpL2r&a? zR&p{jo`-c?*POoI>&)lV%TjM7s9r24m7LtX6~A@q$zh;Di3)HPl8zB?g>_JPAZje& zmlYko0{m%yMqZv(?cX$dsR~1RDockm(B+gOH+|(???z)N^S4U%9TG-*BQ29H$>(A~ zz7hN$m;29klw}4z&-Hj~fCRk_LR}8TM5xq~9`B}56+hEAs+_*a8aY0iFHkkx7s=O= z4+XLfN-{D&FfcH%vf54*sMO5cBe$UZ69PaKx18zigU@pF&DLb_zM2$Zi{KYXsh0^@ zRfoEegHRL>4i1i5o*sc3%E=NPhUk@2E^S^)E_}K8X``c0iGN?CrS#a|MhwSaX$DqI zbkEMxl79s^?LTON@TGtQ0jMxI8{io3?CdG^{elm~!GNA0{3~1&597F~oECCfLmq$x2t|zPfM~%@yIVjI_QN8b%YM`$B{{9{) zNby){54b06#rdNDiaS#UNGAQS{|f*VK*~^2QKflZ&YbUz!C9a~fRxcua;0qwD1E)w z2)~>?@LeimSV&gVUVZrWb?F;3{bI6~(ZLyZZCz@$S@CSD()CgKp~-Kw9~O3UzDWxJ zVM5o-ud)^E{$p({(|_CeE^krDR5)Fz5(-UBU$&_f;zp|gnVqnK?~|ZEpPX&xprZeHXryf*X}vPU&UN1GR>D&E zz|$n@BD0c)nBzJ1w7H>s+^FLH7MwlQ-uhG&&m+5}i}x<5g7C~CSb8vXM_%v`YdG`` ze!01Y1%NJrpb=2u_U}X4&Bv30i3s_1|1S^W7JW}P4tN(<=Eq-yY`o}ME%J0)s5E0KHf71e;0<|nR3PWT(&(@7}H>4?m1$*^+d+q zcJnz%yPnqOrQVtPhoA4Xp!}#MD78$a$8Zs2NV-gk%~mKCy8WlpN%F{Bik+*`^t6_2 zk1frN#a(mO4-%OK3Y+6&V?`m)wV*%&kE%Dp^6vFHIvu$fWAl1Pq`MwBp$44H$FbLJ zG9gsES0L8k-TP(o;WloSPK@VKzXg_X5RBH#x$`H>K5ZYJq^v}U0pNc$+ErVcR21&3 z2sxM`s!U-y=&t7Bi~qILzx6O%GaC%wV2O3kRXnsz38L*Q^b~bZKD*_KzS~K1LlfQn zK~uLHc7x_k@aIN?5*Hu8dRJXJVhwwGA>{i5`nK?kjp4 zDmIswbNg2BpC_9JRo6#taWNv+`kKjHSZb4@Wwa7~l*GqB^dM3u3x~RP5_)g-y%wFK ztE>MZJA0;p*w5bXv6sbrakJ|;`G;SPGR)-vOT^rMlBDFTg-n&Icte_Lr7#-@#TP1~ zR_~BFdOXnlCNL`hT$R zA+%n_;j4+k14QAkk~asGbbynzpG7eJVWr1;#5DHwk*crvaLm9!tcsdj7r-GB32A& zzVRjH$bQmhz^gxYC52^0Pm;?=XI49QI=OHK2=)CbDq{RUn0xboD!b@!{1`GthGfi` zsE86Ob21f@B=b;Fgp_#-l|m{cDT+)fLI{~TjqaD~<}r=~#2~j}Mh>xQ(Cr_RPQI&%ZkU z%i!+);FEdhgkC&8YA(osud9m0y{6+9$?S}>%aJz7vyUcucSk2b&EJV? zQvFliv2oiXqkZ>%ua`^J9hcfC7;Sk9vNXALtDo6#OLG*m|7>V_%i!XLvyOQ-cT11f zetJS}X~}i!_vn@DAR4N#|2}oB*yk0SY212EExAuPDL#Mee1-_)>&>?vmosn7bY5To zI}p2dcHoOlk>HN)2F!wsYawe2yKAIH2V)K&KdE|3e>S}=VIXf(vHsv^c#b7czJ^H9gSp(tS59sw*i2l{maUVN-K)af{<`h4lepX;Z;x*I zC8a#aS{-|j*BKil`ooWj8$2CFtr>D#+c)D&6SYnQKp;D)rOOy+ra{ckyMz6SWMT1{{MuI)zSC*-x15P__4U4SY-7-aG2EMtz@#Ee`6(#27wzwlogP>{w{+OlD!a@^Qy3L_9u13KpFaP{%P-0|LhE> zY0VUxDxPBeE0fBeYLV5=^~Xyz0t&>ola-TSu5|iud}1{hmiQ32p)}}nrs?{)socZL ztIl*Z2_4TLKMeC*{@z^{srBkl_fFGCH|7@i@#yc04UG})Pn~qw+vK85zcUC5Cd@P6|yW0CZ;x^}AB-t{&eZjffUoLto z{{6c_Yt=WRN8NLtI92y;S<&n9xeTO-^BbiiQ7yPwQ+v;^UZ@Pj;OD~Zr zJV5DthdTMB-EiK#;3Q3E-Q*;TV6w8qI9q&uTILU0?;U%JUJ1{!^%#Afd2{@XGELqO z%iTGC7FW{)nI^BFJGyw$wI)JITXESdPlQJxFh+YIbu2QuBfI?|l1n2}i|Yg=_cnwWX+EL}r~qRe`# zSZb5oy6tVwgoQlsM~zuQTpt4if)pdQd5}rve)PC1)BV)9(4rt?Uu!)kOr7 zrFvg&CM0mROf)+SsZ>sidif?_W}cDH6bO7J+f~Swu=vZMvxc|0)9ae04SXCoZiim)G9R0IvEqmk2}~Y zXapW{Rh?ejeLpQpL%BV}e~!uf`o4KbCbxx}O$OF`?2qBSMV^qJ zS#2XHwtCo~PB($6cK3k4lXWVpZC-XeBCh;$3WMmU4sTo(NR+hL5VTg1-XF;O zO@U3Y#lSUK*|5!%=(OvRCe7ZjKP8^*3}M}ZI}~fp3zXmL4VZs+>2~Q4QQ0mZE(7y> zx>llv;_jU z{a7^OV&W|^``4N!;#TnFJbR}PGi$I=>zta{zzB7JaI%K~N@)7TPi8&0U$uT*4 zJ`8*F-k-@n-7dx9mmQs)xjEG-_%~eTjPh%4TIR{Iyp8>;LJE28?975v?U#xUzDYad zKuPdI?HbRYUj=-8RvEZuzDpC-xrOp0m|eFrtFVqmpDKhe3(PDy$VnA zN9+~)COXB;FJz(?yXNe|5XhubO}49T{8&P2Wt@eoKaXR9m@?qokqy{gW=?tvWJ=nbmN{TAYk9Dcv4FK)Z{SnRFapzZ!3k9BX+28W^vriM2K znNLd>n)!4Ue{L2m&3?!>&Ja2j8etL8ab{zjz0Pk<_oHJrUwJ{M{4a0zn~MDx0xd_~ z_8xruCAe}jc4u}vE%TpKCu|rSf{9#9uZREg{mk|hqk3u2*%I+Cr)a7smfr7FU!%qa zeI}Jsn({3>2+y|j@>=NYS6#@y|KI`C<;5R6>^Z+5ExHq36{VDA(Ot~ArO0mIEcdJg z+Z;Wol*RAQbFWKlXsOSdQ%tq*S*u$~&Dpx5+Lj`7h)E^*NSk`JbDj**R_4Z=5b+b& zB*M1)=!YZ(UeViZ)m)hP8tRp{*z32H?Mal@&_3psM(*vMx^HZs+ucVw*?BP!%X58x z)XeW{Kbw_yio3CbX)Tvog>gjUs?tiALzPJHS`?FtkbPavwlR&I zM@z41$e&Fn|0vHg`y#&m%ri0>E*b&S9}kBmSN}EEAbBYl|wH*7ZnC4v<^wj-a>A4RoIKB?{kWhG}f7ew-(KjCo+PpyjURD8r;Ax3J ze@+F)IcHg_w9^ROdh{?>>9%^V*-!m5obd(Y#hDd^U3clOWToqbi5E7dHuX+Pjnouy znV(%sako>tp134*TJe2ud$Oxc-wDMS^PAs%&vln{v-FC$JTj7prPsZD1C#j3y z_(G5U{IL2nd1r>~y30MQ25laeO?NgCiiswEgAj9JV>herchv8~J$C=??u~mA%R{r! z*YK(AT4XSjAg2kXEPcre7vaQQ9iRW>Cr_?+z5CeIv@MJ5z!UXf4Q7jb3!N-_-h}Ur zeb#j97_02vK9_V^vSHBNZKKO%*RasQOX6+&6o2fv^)H4$j(R$ZKF@qE=pIy|5g0zZ zD=GYUTXx}l(|@bg_qp! zKQg#k3}gS`c_E$!!c*l`-T0<2U(CvU3xk5FEhwz+>mWO!s*Xv8=67G=c7gXdR^3bZ zIdS?tDKT-LB6{dfcIPf87B!D`+N|2EYaUS_gMMukKD~F^@JG`P>JRGiJjRl`v~KzG zS10-RwY_;yc7ro;t)un_Z?bku+ubso+5I#tYwiNi{?4-h*)hvUq*PH&Zhf593$c+8#FYT~!&%lmy*{dOR+OtR#jUl65D+TYmCZ9XQR89XYg zXI{11jV#rsj&lu)joni@|O+;^4|21-7c~bUwn7!IXha@vHG+ELUYI z)qU6B+T}_y$zQeo^0}+*M)94R@1i#e^pgN)BD!(&OG{wom!NZ;en%DBuUVueo7A*%1Mcrvy{eN4EZj?VR97ayVBX-yoNKj_Q6__x3MiE?SYN&zFYnb_Q?q55>**efxS5Ykdx8z4qAD}Sw%)qo z+OBzP)$4iv<8wK0Qlq66cWAyDV6&*Xk-g4Jbx0@vV5QMsm*9K-C8B%VSLBbap4m!~ zGLSzR8-41WTdVfD4`n*`I|xF9sFkqvXVA&UCeN--xP+?0^JcZonbB@;m5zurc-FN? zx1dLpt*htz-hI4&e_DcaBr7y_{d3&p$E%6IHC#7h^6WC7Z*x;9vTa+`1S zG3=yo@A6lE9&f(Poj}eX3wE-qs%l|jAwggKsgd*F);V>}E7ROhik*I58FF(Kq+=-d zqlw@->~Ga~Oz%c-uA5cmo}xQD-KwZ>a?H|iR`9L)$eJ$iH`mmAcJg)-PyGiJO_hn& z5sUz2$eGnL?I9d!;?FLXk$iB9cm93XfzC`a{^P;--Wv|`kO$2F`u*D`x0{hqaSde@ zUESP>l7x}1vOh^BDQ^b{2Y>LIyM*$Gg@s442l_xIt?> zY5tX22#9`;3XtHIj~wah>|}=~JI4R-c$I{Zh)BGW|3GiA&qQkq9IX(FeXtPZm_xq)rPJI= z%5x%&^29eYGBUE$P>Qi>8Mu8P_(IS|1`+8mIKWX|>a$c?SxHY%Kf^_M^k2GHA*v&m zUoY&_Ra*bn(ZQBQ#((hNEy68DPCc)0XfQ5xPKo7q5TzI5`d6yfd9pP{>-zQUDBOVI z$v~j}_W%L%pFVw(MIEXVueo!XW+9z~aQS~LQUAieGjkoK+hk;9xDQ{gwkK!`{Wl>k zH#h0nMwtT#0!|ew-yeDzalA6I>pxp| z;V&sEckmah|M|J`(AMq#c#m#cYXXwhNU+BZ;4J`ZhGEOUvGrCUuvodru2h}NcN`H zJC~l1oFEVw{;o9Rk6l)!)V>%>CoMwwF$iL`xcThdoN10V=?U0Q*k8~$Fi@k(>rboL zN(g3_lCKUT5LzV;A6`NsGHqcOP+SlN&fOUyjff_F{=AiMi}Rn_2i&MQ^?Q2yoPj|` zR+b18jY`7lvppz31#z~B+eQdiw6U>SUHqNey+?Jj<~4`$O6BJY|LWK;EuFW~ zfFGl_CWblit5@ZDB}7Dmw1rKLjTf4ye*e}vdGhbiq%cz9-j^@=$!~^-ho3%2<&jO` zZ~6Q8FXBbK@}3nJvQfzl7tPs*i%!vO-MW4AYg^mgrzlZXLBH_BQc_Zf5Az*$Ry}@v z*O4QhHTQOq-+Y;xn(D7&YTAp6!}eIi^XJbyI%Xs$3NCgN0zy7s%601dz|6&U<@|Y) z;8QGzauUv{?qPkTDdanUR86hMyx4sxiN?;}{!f4L5^jXIEqm>|va9PnLaS3#Q-4Kf zR(ur{O0mA6*#ntgobqsSA=S?%?j{gMevxWmudEby!I7B2!m!iR`+<&wg^oj z2Rp~W=PyX3(Cn&%!&_y3CJEdM6@D26&KejPoI7_;Pp_q?XP@FwOOgg^a1ZYy1Y~dD zu_Nm7uKWjV}5$)c>ozfr9rD<`*O#}2hk#|xw} z+a&iVSoi0zU#qj9B~o=_FI~QT*~!Vt&h7+1Q?&?zpst5{I@Z?7(b1hbHlML!X0c(y z6$`HR_Y^qsMQHY+x~JyV8CDv~o!jcL=_KfXprSPDy=`R^n^{B&3*rMm& z?vapaZfwMh6^v}i&(BBg;G>F+gqe;bJDc$T6W_nT^IP}E{GUS!C2Q;HF+&;4$gsH) zD?Z;2dGO%Dy?dz3d`L>F+MeP|yi$2y-a=OS25S5)&d!>AxHeQF$LM>5G?-PSrKKsF zpP@GQJ|tuwsHP?(WTB4dy~az;PY+U!{b@Lk6`5q z5a!|O`5`981jWt|9OfY7SNeX3QZv4@v^_vQ$;;bYT}^EZB_%z*(d`QdIA9<$-n`kn ze?QN@Oa_}@9T`|O*ylvyQaU<1QUO$w-tr;@2q3Kn2gnn1b6v3ChfA}_!Y_w+1-QP> z%Ia)yuhG!Ze*fi3Ow3;s!zy|+6}cOK-v&;SGm8xo!tb| zkp%~KGZU1%vM*9A-epqJ{3iFUv5`T7aNr?!^EO_26mg#V^XJdcpTW_VL9mXfH}&ip zrv*p$8~xOxq9VFGl}M|ou2$s@yKH6kGC4W0nfnPu0+0IY)urRdk3)l~tKWr1b||C> zs7Kga|IC?lZ;b?s3aJSz#7})&>>ZSy-CSL#2FuG)pLb!h9VLl9SEOO~xE|aMD-*^V zppMsm%QAUIMa82>pPfp**x#eBqNC&Z@zxdyGZPJ^nRsVwYfd@xt0SYLY;Z>#oAyDr zZG@xOR{+GLC8#MWQ4eZJnlQ>MB*UNJDR1HFx!hmufxPKoBO|w?MIPtm%mS>SIC&tu z#r8WY*PWeN_zFG#Sp3_Hf-3btocgE#Hy7aR*ROqjecj#N_4S&*+X#|-<>XeONX5m) z(+#s~QS4gNLHR<*RBx}SP<9|XjH<@&>FVhb41_LO#rxAjL3os;E)af(xwX_3I}zH9xpcw4yBe zX3NAp%Ez95bJlm@SB1FE$lMBKz%a{VhxyC2l$5a_Kk}jMh(`&U(La9t0N{~)wv&tN zZ&4{DS-2?Vmy%St9*O|+`*d;N+~Oh@t;~7sUT*Fo3m<#L69xtb8X6j~;hZ=-sao6H z*MG!1F{!Ml>Y#x^wa9(9fh|_93yX_)gM#KfOa7Fto&xbDK|!bnE@L_Nt!CG*T~*!P zUfLsI&fj~TMX7otAsJAPsw{3_ckq?C|L%EfoLj9iaQ7K$bR*wGRI#hKceUNn61qj_ zeMB@c`%Pu0ctSw(`P^gamZ?18yym&PTCyc7`{!=4z91SN2Pm!)XN)dD==#Z zI5ZusZb3MGe0*SPu{lQavfsP^Ho=X2T^+EO4wFU!>N0k9b-_{rua$0YtS*q9R#E#(?a{@$vriKlL;;>hkP6houP> zU-`3dhD5~2OBAqYS@B#45<_zeUtixntoN3eUnjLqB$^5$#nBn)Jts$!_`!X=iAOQs z>B<$^4i7(q*EwV3hD*69frmo0kaqg5EZk2C*u5|kgM}`sLauYVT3U0nvy~ckdIkpV zg)ZZGc;Kk=YeSWGZLfw7srNqKtN&D6II@A2?@L1i^Y4c?sPhfMfFiznRa}uQpkV&{ zM+$h4sclerL-*wgS@x2OgbEG>YI@84%M%h3gj_$bq1*=}kL+0}VD?%)lzfG_I669B zyl_Dgb*%^eH%pU}DB`6SNiSa}wiNY11S2?jc-C>e|N4=gzAOZ%w4|iljh8$;JaUcR z(iRGn5$ve#9jwfD3k&ngxqqpx6_%1Zd*TGGM3|hV<=~fpBgBz9B|V*+jqRMJ+?n{bcwUGv#PyQ*C~3!eahIzGLJt401^NB zfL4@v3&ia_JU8GU7#en>;vs#O4%3Yr>(Id9!PqdY7LsbMT;^YtJ{RP^Lv&0`47Bcf zOc|!e(NQO<#ln{>+zEj59=t4idRChoIIIQNvi6*lQ`YDp>yO{R8?6oq?MFNiwjK={ z_*ht2fNGvU_l3Q~t*EG}N9JmI-1l(bJ7bUQ#Wo@KyW2LjyCK+6}!A;skgRQ>VeM+oU7ulD@1 zvJK$sP^O~mBLF*UUS3D@kDWtx(pRruEz^X10pFl(FY5GmpsNcUIYJgA3paedF4@5c*I=ClRkfd8z}~>(0IH_8 z)rb(f_07#2FXb}O)2kdmPMY@v=!T2KO)uoi*B4EJU=9!ZVi(+s-fg6%75d)n?hY(X zc(~Q2OZlF}3oTuTd zRy#En6-=krcZ1v3lE&YD0Nk6Ko9pQbFwsF8KmTKY0+zJa{R#P;{~m4MSU z(%xdlikFe3%gV!5sn17_b}s5-8pJSY@KA`zTsUJ zKS44)OiDttjMmQI&9Gq*0vy)B$rewYGt42Xe54VNi_?bLAMeh0L&TyXUj+&iCs|aR5Sf zh2A0(Bi0sFWM7C+cD8qS8=IJTum5!;es}`G{Q4EZy&1q>kyi6L2&z}9sS@Tz-+Ftw z(G6oD=4{zp@)Kub z491p;uLS*npqSBegPSeXZM2?QJX+LZ3yYwDfPk!QPiLng%ik^J{E<*m1vMkY%Bm`0 zrDRughLN!`8(Uk?-#yYL{06Y1aSA^9SNqd4GttR}*cf|w{Kt>s$YfFvjc$iUboJ3q zZEI_7MK7lDM~_&1S)QII@i}39@p$qeKEGh>izLdXd$$S%Cx=@)I`Z9qojY;$0MP{Q zHyq3vy}hH~N#h!{4WPJUd{}f!5b#ksX>6h@^#`-e!a^_qi64!z@HK&u;9o-9TKS@Z zk%p@3)7aSKzoZGZtT++)L)BcKb;k}kI@nD&qf?$ZROk6&vtkVIf=to~z)euxv*VTa zvs=I8TUuK5EG)!j=j?55ZEb97tR;7+rN471_TN;*$$|EEn`|T0$gTWL2x!3@E-5VE z+%PpTSb%AHaquFQ1qF*3{5f<)0!YA_M(}f<#%aMF%qTo9R@5t{sA!?1!@$zJY-?{1 zI)WzQ>(|r!qV7eLA@hKR+tURg99u7Zs5aFw)9kvtd#y&wAlMKtE-vi4ix)3qmw;(X zzR{)&_fId<;->Npl4v)GckkYX(+&Z;CsY6ENxxX~me|J3tgKD&j0hw=B&{|Bt}iON zvlA#`8=vCaw6rUiF5T@+-udFxCOXRiYO%n@hCYUy^((9Vik;oiTry5EFmKMl({Pnl z0G8Rci$WkdBjfkb5Hl;QkRo--Gwdc{6%tuyYz(KMf&*^3Me{^k>V#`ibTqp~_))yT z(6F%PFJB&)RtqO)WRwC3;Kfc&IdlixQyk>D?^M3dCg3_!`v9yP1_&D>tvoNI`x$Qv z@PB|PKH&Y$&CKAJTfrNk$Qo)^+SJz3feCpMRIQ$H`lt4Ku6lCGRzlkB^R;eYggTaK1aoM$)%~TLAmA_(IJ+ z%pXTKOG@Me!!t9dfl7#8%G&4{ zZ{8FZ6@i+7vw?PDBR0lpbTf}PfsZUdTsx2Bi_ikwXlWzb`|!}JOQQUN(|v{d(5S3* zHUFzo&-t$%K7IDAD^f1x%VTwjK30YYZ7LjgvsJ2%+k?B74 zw3N5tVWL6VrAt4s6!0k((T&FU(U*e|WEq)Ss!hNx>|7wlo?g<3jg1X;iW7CZ+Fu0z z5fsjs3)!TXGX=Gs1vtmX#&*|DJZd|frrE-36@DgQ3Je$nVoVpnoixwb=_V)JS>-**8ir>6x7#Ohlai~<8-zv8oXb@UHda>gMK~n-a{hI&!LNxS; zeBX9D^@rNRaCt~=odC18vWp|?P;i%;3kcn`&0!f8VBnv5D3dXLmp0nIMw}-kc88Fz!ZAwi08e3FC;8H>nYFq z%Zpg_J}+;5eI33@8Y6wXy^o>bnOTrHE8e0r5pSZ zL?1uDjk^roK+S;M_u~wIh#Os`_<&N+2x@#bU=K4)Y_VnJmEW^}Kfqj_{j7$or3yGS z6O)+d@w?i-ex><0eojq^?ccw{Dc=m^ku%K7R1gZY`3qkU}{UqvPYC;yz!-L*!&Q@E@MAq@F{qo1p#~ZEGs*EDo#zPh3pp*83)ab zdx3H!VcrlzXZB6@e^Z?QUOjjkXW^hA5WaZs+IC~V5KBW%O-f-QiO&G6p(OtZhaVL% zLiX(2*YxyYRbyl0;2@AmJlgBk@+tZ*qhB3JC7(%JBcxmeId9MXUGba}p&U$UC(!hU)EksMd_XP!^x-e!> z!;59XRl}syQj&+(JnAhDAPd(H?;7Qb{*1m4FTrf!HNmm4prTMV;})2@Htbvo$MW-& z{f8taS@@_EV^3s%#4gA%E!Zb2YN~ixapE^ry@Sju0%O>2burCk1YyKER?thZz-dsi zW!u@AOI-Kg+`UUB9{qXD`S;Y+6LIr>A;BGhphhhLP!`2-m5$kKeEH^91AIQkW zn#0q)kWF=0lTZ7UrJu{7qoq9oDy^yM+?g|#mjga> zaFfQx+x}}r4N$2sP8IwchYua{`P}Q=9IpiLWarMEB~~BFN%x?~JtE-s8yRt!rOwvM zD=3T%51-N3hcgq_S@Evk+Ie$QuN-dfg(E!{=>n>k&DVlA4k9)Q2(GjaP(#R&V|O;6ov4*jQUv#NI0)DVZXR zci7U&$ilIBdCQ@zRl$z%(yF+oFD|i$thBl-cng9C?)k6jmK{+q?~ZNy6z**-hKOuK70rc17|HB?E}4p$3;YB ze0+R+NOgo6k&5ut=o(L~7!-gzHZ{Cdbm$|y&Y+nDL;^eYq~=0RZLPbT+mOx7UvOX- zZEcTWj|fM;NlWv_yAcvp+*rP7YKlxhUR(aCfg4NaNFu@h!9w9O zR8&+j-23+I`94<>{6gMKQclhX8}-<+V>AiS%fW`fCMW&Y7Yy;TfU87B(}Cwg49N)F zX)Nk(c23GKunBYBH&&EuiE--({CvgW>&gae8% zi-WZ7Ai7}gq2Men`KHF?ui~*EMMgs6UdB!v?TTy|=9})xOGjY&qm|W&)k_E@d}C0_ z`%Uw|=Y51ig@5Yp=^3mP2>^jki3g>n7ofGF%mInt^>PrB#-a+hV$JKokCT#Dz;R|6 z=TbFYxGw_QConJ&JfqPiZe4^uH8ezoNu#!{%>^9AyTwC~AyK#=)otRF^gj?y^Smnt zaG_wmL6oV~cXcg;UZ@-=Bh(Tga`>2;msi}IS|0kbgueD^dipOof$+fi-#_4H2|;U3 zKJ0Gu63>I+u@w~ho12{wlwFoDTv{exM0^?XRJbDWU`y6y;Rv;s@1OeJTLAQ(O^Xjf zfqd^+fKy8yqQizAuBxUM(J)FPu3~C*X&4#95E-bF>|F+eu|&yTFapqejIp}(-`d)? zGc%KT1(J$D-(1g?{a?_JqxL>$7d&QaaM5thU~(}ljD$9BLK1A97(Yu04hR}+X>BdZ zdx1u{HU0hlb#>Gd0oTC*l1%EoFi6Efx&I*6(D(E<#PXr>fl~Uue4)>Ja1%iHm}|KT zrjA%*@`*@zJoJW~B61B!0*SchDliO$o0xkpRND`QAFPwtxh-| zgidaxh{ot>U}r7WKdqS<^#eMyr^f_36CyY;0Q811>(Vf~AMc_F(J`UDQ#pvEM*FjN zQ7NB0DkoQvpAS7m>YfA?{htl36lr*mgT+81rK{7$+w=R&_jxuc=u{|B){ zNpU3(Ue~4YlMpA!4LY#~iu1C(E-T~pzN>SD?#=~A$KStxr9&kRT|+2?)OHTRzvf6i zG(bT(A2c85tA(Cr7iVV>14#E!ra?bM2pGEOermo=UNK|wtk~*-&R-bKH8_btv0rPB zldGw#gVF-~yVX1Q1UcOu77o&=AG$U=zONbTo>#&9TSJ2#zToKy*eQj8Vatzs#HgT_ z*Hb`~g7l-NriSbB!ICWCrj>~2-Zn@oycopzu|=4NV<=+|JH^4A+XnIR9|feVQM&)| zq4>UiEPN51Km!gkyCwuz%R7u8Ge`;*ZNKbF;=-5U_Hw)(N^BK~P z{@+v)78=SDvF`t((Jcyjf%oNLwMr!T<9Zl2cM58mu`#NF=~jhvC7S13RDcwTM*dTBJX_547c*ckfQSHry;LD+3bA%F0^a zNwDS?lG%H0?9&b)ZeLc-1K70F)6$7>|Q8h8*D4|=+~ zzDqxQiCu@bk)^1~iqIBKVM@r5p&%!RdJ&Y-a^_L;TLb$FJoP9@;SToX>XOXw&hGA&8g@%W z#acR`($A5?|Jw=SWZ_*xCP+W%&b?e(TH2Xm8U#k;qSfTLH?FelYb$ezlq8b8;+YeV zgzyyLvXBx2@7_}}zUj z{{NiuXcw&XyBF4=0!aD;AuoVrSlgCo`dz)Onmn2gEc?X7#10=e{KmOcGYiUhS0El} zFQ{BbrM1(($t5MvKUxJ_F3ISDB0zxh!TtL}<@(csu~K%eAdAtV8}JMRU9)K^E1~aR zs?mgtW0Bdbr8&XPlt8G!Vj$?<8NiBcE3j1v>!1g4#H&}r7Q7PJ@eVRRK*P|+tt7`v>pG7I@ks5t2rt3vz@m><*6VF4e2L!?W=&F$B*Bd7HkuLxYHbVbfCm5 z@vghkp#f-Kh#MrjIYxTx;?XVJX;^xzx4RMF6%;UHi;14MVMO%Y!y`yc+y@8}>-7tJ;;1t4w*l`Gb`-j^#k4`FP41#A}J<;dW@*1%!mw)wQx) z37p4;SV~JuR5Qx|g2nl2D@BZ8CS?*J2$Gp89$HgruS{-xP!HJ%w*AoRDOAK zM{e$k?bIAQ@Bj!%Y>jVTg7^19*a@7|i|5asU0lFp1a}5#llDJ!F0%?L+@52jg`5lz zkHd;~0cB;1aJm76xzK*6Pz&jlWd`Zz;B%F}dpAQ$7z7DjzOf*I*y%0`dxsH`{ri6- zN&;OAnD#6?`=AAtv$E?QeRcKPdpksDxWetW{TrANGBd!H86l|-*9q43HbzP;?23%j z0R0Q+lxRSKpMF~czk*bNPTIlBS}g(x5a^l%X^@QM2li+>j0j0eNkQ3wddM7*XKav*!tt45@i7&gvSTG#(7vm9>>K!WDn?j3jYbT z&Z-jVFC|0`mltW_x*$q#QsS8n@!RANSAc)|x3hdB58;2fL}2}Kw)bjpzRn)j;W>gb z%}1_YPe975pnx@a3H`__Q=}U;0-RYN5ePSefTBv;BPgDI}C?`tMo%0 z8~IEK&St!Qi)&X4pF9ZlxIz;B@Dv%G@Cqy<{V+G01ZX4x)*}5Py9p-$TW6>K9*~LbnE1w5 z1(VXsk9%0??yUbY!-62&6!{c(q~(dt&7Iq$xb?EN_2}p*@oHjlZr)IpBJrp45U1K#!H^L@Ogd zd7Zizb^?JJAbI=~TNbaA^jqi}ctwbDmX@09=zW(7w`wS1qFqj+k&cw`bxao;3*)iW0W3xn1Av_)A$_ntu8gj zZ22X1CJkT+VXmqnco%O?@;s_QGs?NtCSYOxK}>p zk^8rv2y@XS=$U^Jz18=wVLvBLv-6N4Q5RMY6WH1~Na6p2p;LF4!x3nY0 zArOvb!Q6PWgmI3LkPv0>3m(nAh@XS^gx3pMLB0=B_{i*coQyHsZi-wjfP``6-FPod z9xO8B(Po9!Kt2OJIM{FGD~*humzI(7TWB}LK0&^W!gH#qgYXWq5Q#5%5n;6OaLI>~ zR`i~?#Wsbq^<>aI1kMwSiS@C$d>M{=+4PNl=phgCc4ahD_hr?)`w&2c`+oiW`N4UN zPNlxGl006Z7;kl8O}^$A$r6KQ4hd=a7Xk>UMP4|75`aIVYo3lqN^3KuH6)Sh00#g>efA720?;*v-kVW=v21^f6<_mB<%n)cf0ZD-IKn8L?DeFa9 zSj@8Qd4qMFlD~!PRxT#a)ankTRsj09O}PFP6zv5W3tNhJDl_Z89|ltk8E3s_1Zp9^ z2kNWKBVjPn$fB6(VS?#0L{b$K6Cql+J|FbLP61aW36>lPMCvmjUQ>*;);Bk{`ZO#p z(hh$27Z(&3a%MH8Uq4A|-TM3IPfz$v2n`*Dvw);QSs234G(9%(v)Oj+fMC#HU61qv zk7@GG>LVa7ZcOYUV8x5C4c)>&Y_~e{S0TL!pFjc0J$m>Nfh_=G87ZkX%#+9!eiZL? z8aZ;_^Ls494M!rnpszj;U1O85K;!dUodIG9CxG}uyvg%C0UOg=a_FA((QH)gL{4SI zP7WAo81bZ5HcCxJc<#cH_~yzdmn3`+M-Q|GR5TkBd=MkSug9E;F=kw0)#py1Ru+sP z1?tJe_2{V(B@Df^!Xg*@T4i%SyTT@6)@v zxvfk+-ctZ47(y7NbU<|}_~?-%-?#7=Yinymg-L>O_x7IYaY67QGcD~FCK29Xjr2Ew zeH_Cs5%#Z@&xTH+qa@eR&?q5T$H0+6Wb}Z<7eKfVA8_K<3xJSBE$5O>P!M}uFcsu) zUaPuUa^2Mx4jlK76T9~Ar9JjIS_u|Qh{f(H#c{v;UKi3cGT_sL@bM(8mqij7*zj#` zo5z4#RcP3{M#F!)UOs-D!h%!@WME(by&rYM2io{cgwVVK;L2uLe8oAzysfSR-puFFlp$tyRrwE zkjbl))oaUMPsQ~>*!xnr%f+}ZSgxT4L&cCd)1Z?MiQ_h!bIzJ-YBosiuHxgEsT7Nc zCA#Z<9v;z;xs{nKg7IKDa9STJ7Bjb=bIw&$**iFE&G!KY++$}_xn@;=ko=}{${Ab? zu23`S$0irI^KdYYjkV4E*&M5TDw2=$U!Lcm=vHk$_&A&{pLFKUaWyY|aJlMjHrDMj zcwXC|T6;%HUqv!=*Sxu62(L%N>0n%+W?_Cl9LEq?>#4VFxpO||2o6zk*nZhgYB)f{deq#KsfR5bP-PTv?88J`bxO#|I=U5sHUz6mbQ-n;HnkyI$vwhEqPdt zzb-9O*ZLb>uxqNvZHLD*g5j2HX;0^jOL*z3+!ofJc{0~$pPOEB*^&_~>C$q=P^W~Z z!sSeK7uS)N6NlxPTHVYiKk|>XUk5!rZb;4b^-D&6C=ZY2FZS*>lc?w@gZhlL49B5e zs>!EcyWWj>!_d~)XyP)%mlnmn{j+0@1~X!0$&V&XUa`pt-}aj5d z9_`xdNu#&_)~zY3*cmg&#gHi<8a?eBgKp2eET>{8ilTfo1A>^^UH!gNZW1z#{r%Pr z&aof0_5Zr|tZKc9deXb1@np03>(^t_c{JWkiC=%p((BuUVArT+vYhCqI7~yQ_AJfL z`htBE$JEO!W7y25%x7FoCNE2NxqLdbUTzdy!5=HI!N8Ic6}$8En1E^lZH$vbf|9bX zqvUG3c_*_Q!^ON`z9q8qe)~lfiogGD?4d4UBU4eO(~10eAp5$ro6B>><)o#xo7|Ll zR8l5mdQ4(%w=JH_O*kMtaYFxY91!C{y)BNQq5etyON=m9M_|z zl)Y%MD~D0;rz4B0u6_H0$xW`ROMjO)5*DSbdHM?N2ah)07B2ym3% zsXuhZ=Cec}uftjTh%Z)U`KjZGRdE5Su zXH{P_&0DB*I>%VA-OTqPAwt^GS7~|kcfY^US(~0CGsQaZQx$5LlR{FtS@!UuH~KuKSBiWQDpY(I1$5KG!$C;+8lfLm46`-MsLL@$l{^ z*Q=igdTCY5Ob<-XMjkFr7^Zw=dm_9#h>wTrN>s?f5Uz*eHPrK>7cbiBFlZHN?Gf>_ zz9xO8HNP}!zVM#R%q{|f{7lIK!RZ%IUy2{q|3zIV*TXFEXutp7AC0zmd{V0nEX(rF zGcp7U+PvxOxk0Y%W3N`fsi=FfQJq1hREA!nf?~LGy1wSbeA>Jy-@@$OS4vz4CnVH_ z*$$Adyp}9)_3Ej&Dvj{P(o?1u$!9v>IjCJ4-e)irOWnFDHWGgsO0g-N+Ex3tcE$At zwOdwvhUiYM>Xi} zvoZc;*n9hLMD~4WhjV?y_1hl>wsrsdck!Js%Is{kV>&Rc4_c$TRGP1jY~5n@t+Q>% z?leJ$i!TlKaLl41V%i5_aD4CL(<#*L5dMLc@>7txwgDc2P)8OvK!Kq`%K`SGqz3 zt3|TOQg=e2a%efn-ni|`M_rFt4IjB--o9o3MXPTUA)<3zw@bEmt{9c{y<8oMlRkNT ztNyD(TXKQT8LMHb2K&nGTa5{~Ot^C0K6`H|{~XiC+t0Ga8MIRUqW<5Ep^(Q38!*`%ddu-?j2&hAUXN1AZfTQukX(LdYXN5 zq9(0*R7ds>Z@U`hQP}E|H%URON1QOpm$Dn_C9C1vTjVD?NjB|Zyi%IUS@gbMsrFJk zIgRScn+(ISq#QlDlOV6?Qki z&VM_oC?V3MlA^NI5Ns~`DVyH>c-#G&H7 z{Xeap^`W8qv`9c;-#Ah%1b^O?ye zG8l_Cdv(t{wqL*y3(=scMt8~donj-oC;U8U%tk-_Z2rfq{(-bC2+3O~?tsO%2b3gj zOICHQod-q+8qZRFi*Vl*f2jJ&KVY@Rl7|z_?Ct0)MjY(3yykX41 zjMVf`ge>DV`nd!aa+nRDqMUG$jK~}dfo)4IHJ(XTZmCDA#BRnEmN z@k_Ub*CO1V>Tc|s?nUo&E>j>x?`C_J<;AaLVj~fqYvUDlXV=F^AB0M5sOzy`MAn=A z2b=x(WBU+tI`U*;)AB)eHpAo&Ozt4XXxz`Dq7B)n8u{_RvlGJ@-0~Bu7xhji&GOS7 z=SlI9GGkfAlWdJfp;Sw+1Tu7PIeYqE()=P5I-LQ%v%R~SLh_kzR5WSy&c%gw-vS{d zq1tnG%05EnD&NrBJ~y3dPxNdl2?t@AzC0)1gVzYMbVXIdXZUK6$W@Uxb+%OMLSBre z3Su49*6!x+cD&NbXJ!wj_{O9HhsKzN)*oAMfA}!Gf_*je3r{|1NLWvg7rfz;BV}~N zNt;`qSzcgCo#R43Po78bcJ7}n_H}v`Uuv?hmSB%PRQ%8-tje}f@G)kHGtN_OJB2ls&tOHszMrwNW#)3C zm{n(S|N2y_7A}%~jG`8l&rxU*sjqHvoQT&fJG(#`e?u92_Cwbb62zqUoz8YZR;=KG zQ2wBL9dFP@lb`7hx_|C;3@Edq<2^k*-NmTW+>1_1I!i6bF@p-x`j?|$%p`f(-_BJ$N&ju8&W%W_MRzn92Ika-*+f*+ z-adW!hWGTo=R;%9*R{XGIW8hN2fn3m-5O|C!u9JMsLyl>g`(7uDn!+ePl8|RYDVdB zUy)%$3H|LC=-?u`=n;cO<)_Z*dv(#;Ymv8yrM}MK>{vwd2{-`)&iy>c5~1}eyjkRF zuc|U~SD|L*J zKQ?cP)EX-?w;@0t1q>Be{$#^KMmm~L?abF+-{$ODi zxK^3LIlg^SNKz&?C&`qpB4+5(V?ql@co6G{W#wS2j3gziA|}W76RxV3^E6q!g+fF` z?;dhKV~M!4v`rkrCicLE{>j9t%;g==M0pIvZ>625__8Fe(Mhhu4!6cla+JgnP$`_| zlD4zqhd#etGyDam4@V>=5k&rpou^ov#kc5!9g7RbG(s_&kI?p>frJ=M*5ae7=CB@W z(7W%l-g{^bGK$c51v^?=zq?#H`R_H0A43PuV- z9wqzjUm?|)`y@V%edMO><-3PVIeCOfyu>JP9SsKrp*Jaq4Zj?H%TyR}5fgRcsez7U znFOH*!E&bY%<4PXYodBuF2YCc;PQ0RYVp;1jg3_4c}WaoB6owbQwrJ6j_5HF9L4To z__GT;zr2Y{FqVp^xE@YwtF^TbZ;>x}o2__*7Aa>X{+>^%K6XSTh&KoytwH5Elg zVA+MZ^2WVBpXL=l!9cLxJFMgrR^`0c4JZkT3nkbKg>|C5cNO-Q3kpgzEg1b2y{6~j zijn-heqT>Z?Ux-U^7@{^5AD>_0^#_ZrbcI`{frI7v3e-~Li+W>f(pkse~#>o5Bpwj zI}QvVylpCYJM6GBpirq@T!Mu>)!|rKzgcPwk6Qtjl@jMo&|w}_*5<1|H;Y4>?Kn#& zMeQ4#>zbG5>G-x+G(D$=8q$QXs`%-nuk$7Gkzzv{ehgk)O4NwoiE}xNgrHx|OLL`*h)EqkqF$k0jL%oh6ZGMaA&afCkNSPO(tGVV zh^ApkT|M*9?9n4BW^+ul$;EeTbkvp+Q85CnB1Rd=T{KI*7oS%4b?+cSOh3OE6`F4K z@CTNuvxwj3<;@m?YVF0r*ny!ap0hFXz41kYKzCJ@JQn0Bg^Qva4QuXLVv!y8(<@2} zs`s6jlu1xwlksGX#E3z!t8y>90BMyQ;zy*+QHpnycDH=)i4a0-f#Ln<3w8MjZsmUI zrE1P2dkb+P7G}$Q9Hgcq#^JZrNBOs|>2s2jLH6n)X*=Visx@rHr&n1%VdbHe#!A;V zsl~qak-AVJM3Y~vvTjSS?A5;;w(AU4WsOISAO|Kpk{?rsr#YW!buX0{4ZSSmuYcrFyj=-CE z`?t(@Og~B`iwABxTbv^o*u|YVQ(fW<_bLfOKQVltyoQ0OqS30U3@?hd6G86%ERBE zB;fnpKS@b-`LQwtbkXiRk^T))R9Ggwm~H;WtfDEcpB20+A|2zyxRCoAG^GBaiK<&=Q(-9?ndraEzF`V`9FyYw|qicqRp6UgBWzR(RPio0n z%^DNaS7PWTs`y7%ZdBZkZuALaR}Rxp9$1bFfkj=1P?7)!!s@}wWSakB%GWu0eE==P zq$om%w84fQ9V^zjNQNd<^vDc*)u5XqHnuy7brqH2gCCNPf7k7k@tMl_!wr+K)5Pc3 zf0Mv7XXILcvly9KQ9zhUKOjsqw^EhFi;u!YqIPCoPl9|!0&vQ+S6{7cmI?VU`T(;U zt!QBM6%%2-oZ#^OG>SMs;cm`iK^GTAQ?pV}8jnXBpUeH(W(B z#2IX)-Un^)4E;<*GD^P0U08Y^{iy|d5kab}N+WW=Xe7FHH*exDRHymk__3loITmQJ z5#(hOX&t9HU2~&OER(Kq{+JfRD`>$nf%)jIR>&)GJrI6jo@6`dG+Cm zo*$=2^_HjN4<3u*9RJPu?--kWNGjXFHbaPCN)Arnje}eNmj&>;ZMxGbMQu!(gLA3B zzus&{vpTyZtriP;)KP8W&w5OYlvPn7Ptj->=B0@_`ae`S7=D+-FIKP0<6kuF2;so3 zKO>S0D(`Z5d9j#nLP_V%TShC@-TKlC_+773@B}N+^MArbT+nctw*DGvQ8#cWXHa%R zOddNoOq}$kUAuzZoz@+aCJYQ|aA#vL40gMQD41P8BTCRkg{UaI1tNh8)uL?csV;vH zJbbg%8xKby5jzYJ8bmI4%w^(!Hs-J-3GLVy`}b*&P%!g7H!OWu9XNUMCm=1X|_bNSLjg5 zc{aO*8&~RRS@hLvhQfwqVMPUbO648BXmat)ASk9c;SQEC6?cEl*bl$^5MhL%f2`3ms87xILm}YP8odpPD?z7aRx?DN4gah^}l= z{9w6u1p~qHcxhvtz#ws$JlW`y&1h05na@+cB8|^6?A1Lc)M37S;5v6TYYuR+o zLpD3;$xYp#+7BJe#BOz03$jU@y?cb?=}#n#lu2T*u};5m)_3A)i3Ke@k} zQKI|d0{PkV`R`Ld0*<6V9_0-^=XCsCzx&~J2{h~t5iwLtQaxi-*B0?SK`I&;5JVT! zr>c-(i&e!ec7#F-1!R!sNkL{zMZZ9m63bd#CMLYs2OXakZjw*beVI^+*Wb^N4aa}P zNVi%y@h~rIt5~KyrO(7EMR7{lN@YavWZ;3$qK8#ApP&IdA!4$RAHGMuY&Ut}CxEFK zqo#dUl@``OcjaL>Gon(oZ9)^5wAb_ai4uZP6<5{LSIF7=xY7xHxU)RH1ko=QSnEYF z`&caP_6&tsw%`SsQ6Px-u3vw1)4oZC(;bPLq~gA@g10$r5ph^Zb82SMoyn|sVIQsV z05->h*X@@MCyM%+RMX$bMjqX3^*8HE9LacQZAkYa%i2I9v0hiv38BsYRgbq6>1wV& zCDZV=y%aB%O^(LqBAxLq7L2A0VH5i@uin?p^+ce08!}paC2Ur}4&xn@v~xmE|J8o? zvUVeH^~|UJj18O+54sa{3^DtT|Dl7D?g;T!q^wpkIdaT~=2F^SNvQs6-078V57cmx zyQ-GKt=O~~i>R5Y9)^B->B~sXO|dsQZrCwpPlYg$D)KD@Ev%Cj!OJhOETfU8{8*u? zV-t}QT1`D%pA&R3A9E2&%gH*$*b>6MtXun}f` z4{3`ArG9dGgHPd!cfx371WjlR+=RKRGjE8q(?Nyh-*a1K#Uiq$^f-MayXU93QcZi* ze!|eD;4gK;;=_lw31mM@zIWT4&U|5*FVuOB;a509T|c~CbJ?KALNoLRg40;`Q++!t zOXIf(g?QMtG+2%PQT`Z(Ug;MUB_(fUFLYd{x_-Lu$qMU+zjDF8`{Obn>!z1TwrHsM zhEp)egXpgpzPyFuCsoVDIj~)$Xg`!wgExjf%^0a0lK+Af1ScMxwo z20n8V9O^$f2)DM4ut3v5^U$mNNXE%s8%@vL`QY1Xq2Y?!H&rLR3QGQ-z=oyVO!N!i zxXY=LLNy~M<71oW+T;T>@=v=jXf3Ag$Eu@@(lWADCA)FLyrLz_^VM-iZ4DSKiA>D2 z9Jzc1nYFY|#;*qsWfvxnl9XQPy6!M3(7nZ@>}O_vDsPfWeK&qzjjPbAWJ*zSkzi(GexK{ zMhnmHEeE}zvt$L!^NvNg_y^Q*5*>Bl9SXf;9BS_1-FiTvc1Ua%fdu=AG@hf z{*I`agy@!2Y_<96_x=?U>kwPgBapb?HkYD}X}m!)Xkn^p{JME|@?jL_bb+SvzPI}< zsZCA;HFx>gufdVCPww9yP#}5N#|0y|Z=3(oe{y2YE{y&d8rK$xqHU7RZ78R!Dl$(s z6*$Tg^ofeClF;(_@NH(2sCXomhGt*iYjZ=gfwC<{@ z^9mNO?YzEu{lg2(T1i$^EEA^>TifvEB#URG8|@hF8Mj3;xq?|@Z3c$KQ{U3ZAKfQ_ z2OR}kANXxCn_0|;MJB{MQEYwvsm7z44@w<$R>;t&`To$|8ff^{zgD?dE-oEqgkJ+AOwNay4BuN*nSx9}ZkZ zSoC(^pti*uJv5v5yzjNj+beNCUV1q{@^W4mgYLNtIhWPY@W-N=Pf1d@ADyjFnZL~0 zJz&{-{n9meBal?k`@O}Lm^;@P9m^Q*sS(Vn9tsEBqb70KmWsVuZRL*s$Vn4V+s5bf zawo1nu{~S9_cZfo`iH<;tnSxp2lbX7DmM(HYF@Y|y_gg&SFn0qmDK!^A{fhG;cBC; zwu(2l8TK-zo-hB3$cTcXU&4Lur@Of`PJ>?t$r_I8{-k!{ zl)be-ZCecOb&us_cP(dgDWNwB4Tvfnj>~)alLL_|9;tTGN>^vsIQ`Kl^Y+d?rpGpv z0*M>E$pXgW3Y=K(VXZj3{MDp-MdDX>LV$hOjmkh47P!8;=+ojrC|683)t!azPTXmtht;2e|H-2t*XqPXH-XfZnB|g~Vm~XA= z*@wE|HFvE`(u@RA>~TRQM)^x~1PwzWH<%x%`v~6la9bf2N*W`$t|9sQ#f_lg(nx)# z)rs{090c(>TxNNH-^$DF&873Gjl|F*C|d*)RluOdDbY&_IKv?%oX=iK9Q)JMoWY{a zELBQDSo2-bNx|KmLwdtp$KWjj&cR*l8%n$J{_kzAWy(=csR$Csf?eOjqG>3d*5Gqi%CgPTreWd!UAO%E!UO?ToLwnr!;V{nAwd z?@rpm13}2=f=vaY&x@B3$k2d{7a}&@gm@P?ihq{VGNw-4_Fm^G&d))sp1%>ls-2g7 zq4@g4{nj;Q-y=i_az5Zm^4^@5@dB|^I4}-wymoCLf$G|Q=@4%T*7o1W}9tItvNi1XWk}q z_rbeds|-H(F5T3)lH!3+P3~T#HlZWUQ{s}9`fKMYF2~6~YZH@=y#3$X+9Ih%_KH7E z&R!BOL+5K;i%n1Okq~}Pl*zhi@$SaeOl#t+8~y=UU23uiuMdCSo_rEW!sMD6JSpIa zL-(kKcy*u}N2k2og?mZ@LukpRC-dkb-7At9;&HNBM%IwCj4!^}%bamFgq9N@6}b0F zzs;ygYVa{02dv@5Y)9ieplZ~0&_|kd^0!jez)SYxwC_hzi70{qYy&9J0?1L05zu@G z5>eprt3sm9-Q9f?D|NJUN2RPAHh9L;KynCv zfdz?rAHVACu8RiblnA^$!T8_*B(b{dKJWS}6&^vw)i% zB7P2hkTYIdTPyCf+Q)!)K#WNO972E+OTUkzS;|Zf$44w-oe3@rfMT~`vlKWeueAFh z7U&lV6UD#e<0R^$7zo56=PsSUJn+78;@d)ut`R`Mo|#-bKitX$a<&j14wW6Ec|Ixi z7yiKZ$GlWz_s-vp;?`n!XHQQgO<*Ae=h-V0x!}3-ozyr)Gu{) zo)CsUmUQ!sFE_nI`LCza(ZvDyp2_7HO+aHn;1j@x|DA-#AXGZfc+&`vWDt@YIM$Cs zcS1n6Ln2hpAFegTFDr{d0%B1lENb6Fd29f`9{Bpz-}f3a#ujb_;`-iOVAxT$)xj$w z%Sx;*PXNo81UDYKHUR%H>c7wXkuFiHU3h?@%&eC3`uE7v<0GnXp*%@|*mvo-aNp@p zKt-x~HcYy-c;3$yNUQ#ZW*^3tC)SSw7_^U}3~k$@qD<|MD5-Ml z?&~WD%ONDqj}eMx-MEFjKlH!ZiN7I@=nJvXv-P%vNrzCHADHq&!$4!jJo*6&Nf z$&lm|cPo>cf^(`{W+DYdEbu=Py%z|4BJ?@BUZTVWn@Zk2=tzmsDI}MWUdcld0elz)Yd=0aF|2s3=;2Z8v3P|-42A*V_;z!dkmk&U z%-xI6Xe+0^Kcg^@kL8hllLi(8VyOWE0SEq*OJ1#DP6K5`N^-In1igKV0)W7I3Q=50 zb^w}jVb}y5d00}*XJb8b|f$|Sl)8>zCM?Mp5O4e zx$n~wZJOJ|<-v9%3`|k`82rxxm;v5K3_`(XX1UHl zOPc`@2GDx&0U=*b_QnC_#J+eDbT~WasbIT(ElWRsn}C7`zO8|MJ|FWY%F@rY1xG?; zHQ4k}=%&rkR5(gh5RAE2ULPaR4olA{z+~BYcH#&l?JlTwzzVqm*t&&1N=hD@3-2!K zycDIDJJ;6R{v6m7c!WMZJ+1fAa?9Ls|KQ-ZVcjl3MexGka6o@=?vp}W734xG(C=Ho7T!(<697+6Z625SFgi)hUDvj;j$_m9luVJHk z8-WjA1noJ<47Y9=FVOG%JnTSlNjUtfff)72ImXnzVq z0ZKo(fdFWx7gsf*DF(a^lX9F@)fiZAHqEVM!OU_9SHzC{y#53z5>pE#18Gx1hImV(IiNxKykIt)LVuI-Ap628GeIHqdAya2R|oN zWbo19ly}y*+Gu{i>ML^rQ*xyEp+Z)l=x(nJWay7KH=pe;^}&HKSkk(Gt-bf}7(u|T zdIU8?I|`KwlUBse@BG}f=TD#BeKsqrry;tgQqGuJ1dJCn zl%`~@Jv|%H2ftsviUhHx@5!n$Sd78!Gt+ix+H`&^-`OcFD(dd)asr)R->h09?*jCQ zgv-hxB*y{2fN}5_T(H}??`Ql>6CA%BF!S+!tgdziAsb)0=rVLf|FSbc zx_}@6R|%un8&1joLT}U)ehMIC&q+mJAPk_}UjTlAimX|pd`S!X&8`Zi4;7z3|FYs2 z-)?89zVk*(W(Yu4xBX4iSfz#8*=_iOmi^ilpO8uS*$@_3Bc`XOp1>+0kwy&o@RPNELey(n}W%-tuo{<43Yz~-Mh48F)!cr7ZjA_2M^L^F0Q zgfhPX7~BULU44ChTWc$U`$vH$s0n-dPr&CM4rMdk6u>d$SQ~b%;HHQ; zCSLNPo^Lgt<~DJcNXWpgrzkwE)oNT3U&vWQ%=>ooX;GV1|?L z0O>f~-``ZmQ-=+rhoIC4^MD|*O+I$CcuK7o5vZ+bUTl5^$#1pYXmjMh+Bkw1loW77 zv6q&98PO7d!q0tF?y|74aRyh(@ZjK|U$t*({#$;qibKr+3MWBvai2{H+mW?v;fcb) z-v#jI#dH$}+}1HtykUb;SlP(F0B!C0^uk{H+00yG41?!2jE!U1be)e352!#(Jy%_> zp?^z<+0k03Cjc{oZ&T7f!Q`6>%6wJfgT~1m&Dr_&31Y>Ab5suJy{Q!xSuhy@Qgb@7 z;nNn0|I#ZOs$fv(g3}CYoWuCWjbdu;MtOb6&OgEG^#Pm)C{W?A z12#^8jPU)+kaznC;x|_#=CubrJzTCZK@s%tP@VUC;NdbXVxe{gA%VK+)_3^yjp3}G z4fR&b^C<#mz!*kQ`2l;@^IwZ5bf83s<2kxOf;KhPhViQh?)0D!gNF={^z?+_iUWhO z{7sKQW6bjgEv3O62iV~)4vyWuJ-AnXXcQW(~+e;-qx zm?$z1a~$A+8w(}Q|GFGTFtirn;f9TmLv_Huyg{@fWBB+X;lP3H5S&RpS% zo#mRxIBYVK=CPOvNS4p#^Rxc`Ci|^h?L9rEB_*!Up7H!Gyo&vmi~l=bKsc&M%yDOT z7d%uzh=Si~f%LpRL>05JOoyd70P6Lij0aW~^w8I@w48VbOF7~vhod;)kAtm|;3$L} z422=GFZ2H`MeRUduK_0%mLNjD2~g{3s*6=YeFp_bg0V9Wr6rsuA^9OI{Udq@#S<_A{j@=Y98-9&?%cQqKXrB6$_#;n_mB4x z1S3jOR_Y-PQdwZJ>OIi~YPq)RgE^|n;17p|`|`}pS1z$4yp8Z;>Tld|;@}6?!ix-y zituh?dn4xXk=O1$xc2ivLfP%pHKoDQIYL@3QmxR7&&rpK=d;&2b8hE63E| ze&TeDjFCsQf{RZc-qtQsQZ`H!0B1L?Wnwd>(F&Bf4=i+8-eXv#QaHcu>Tvv>%dG7 zre!9oF#C{6-nzD}!S54t^~wAbJ5hjyO}JryjH~M1O z%Zcw~JVPmNu|XE6`F~TW>&^N9J#di0v$Xp+qGhQ6KmB9k?2?>Vyq6RU#XESSfGFNm LlP!@pdHMeUNx-X_ literal 0 HcmV?d00001 diff --git a/deployments/xpumanager_sidecar/kustom/kustom_xpumanager.yaml b/deployments/xpumanager_sidecar/kustom/kustom_xpumanager.yaml index 69acf5898..3ce726271 100644 --- a/deployments/xpumanager_sidecar/kustom/kustom_xpumanager.yaml +++ b/deployments/xpumanager_sidecar/kustom/kustom_xpumanager.yaml @@ -27,8 +27,3 @@ spec: - ALL readOnlyRootFilesystem: true runAsUser: 0 - - name: xpumd - resources: - limits: - $patch: replace - gpu.intel.com/i915_monitoring: 1 diff --git a/deployments/xpumanager_sidecar/kustomization.yaml b/deployments/xpumanager_sidecar/kustomization.yaml index 728397536..a72b9631c 100644 --- a/deployments/xpumanager_sidecar/kustomization.yaml +++ b/deployments/xpumanager_sidecar/kustomization.yaml @@ -1,5 +1,5 @@ resources: -- https://raw.githubusercontent.com/intel/xpumanager/V1.2.18/deployment/kubernetes/daemonset-intel-xpum.yaml +- https://github.com/intel/xpumanager/deployment/kubernetes/daemonset/base/?ref=V1.2.29 namespace: monitoring apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization