Skip to content

add projects capability in CodeGenerator #5002

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 7, 2020
Merged

add projects capability in CodeGenerator #5002

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions ...ML.CodeGenerator/CodeGenerator/CSharp/AzureCodeGenerator/AzureAttachModelCodeGenerator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ public AzureAttachModelCodeGenerator(Pipeline pipeline, ColumnInferenceResults c
IncludeRecommenderPackage = false,
StablePackageVersion = _settings.StablePackageVersion,
UnstablePackageVersion = _settings.UnstablePackageVersion,
Target = _settings.Target,
}.TransformText(),
Name = $"{ _settings.OutputName }.Model.csproj",
};
Expand Down
48 changes: 9 additions & 39 deletions src/Microsoft.ML.CodeGenerator/CodeGenerator/CSharp/CodeGenerator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
using System.Linq;
using System.Text;
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp.Syntax;
using Microsoft.ML.AutoML;
using Microsoft.ML.CodeGenerator.CodeGenerator;
using Microsoft.ML.CodeGenerator.Templates.Azure.Model;
Expand Down Expand Up @@ -144,7 +145,7 @@ private void SetRequiredNugetPackages(IEnumerable<PipelineNode> trainerNodes, re
var predictProjectFileContent = GeneratPredictProjectFileContent(_settings.OutputName,
includeLightGbmPackage, includeMklComponentsPackage, includeFastTreePackage,
includeImageTransformerPackage, includeImageClassificationPackage, includeRecommenderPackage, includeOnnxPackage, includeResNet18Package,
_settings.StablePackageVersion, _settings.UnstablePackageVersion);
_settings.StablePackageVersion, _settings.UnstablePackageVersion, _settings.Target);

var transformsAndTrainers = GenerateTransformsAndTrainers();
var modelBuilderCSFileContent = GenerateModelBuilderCSFileContent(transformsAndTrainers.Usings, transformsAndTrainers.TrainerMethod, transformsAndTrainers.PreTrainerTransforms, transformsAndTrainers.PostTrainerTransforms, namespaceValue, _pipeline.CacheBeforeTrainer, labelTypeCsharp.Name, includeOnnxPackage);
Expand Down Expand Up @@ -175,40 +176,7 @@ private void SetRequiredNugetPackages(IEnumerable<PipelineNode> trainerNodes, re
var modelProjectFileContent = GenerateModelProjectFileContent(includeLightGbmPackage,
includeMklComponentsPackage, includeFastTreePackage, includeImageTransformerPackage,
includeImageClassificationPackage, includeRecommenderPackage, includeOnnxModel,
_settings.StablePackageVersion, _settings.UnstablePackageVersion);

return (modelInputCSFileContent, modelOutputCSFileContent, consumeModelCSFileContent, modelProjectFileContent);
}

internal (string ModelInputCSFileContent, string ModelOutputCSFileContent, string ConsumeModelCSFileContent,
string ModelProjectFileContent) GenerateAzureAttachImageModelProjectContents(string namespaceValue)
{
var classLabels = GenerateClassLabels();

// generate ModelInput.cs
var modelInputCSFileContent = GenerateModelInputCSFileContent(namespaceValue, classLabels);
modelInputCSFileContent = Utils.FormatCode(modelInputCSFileContent);

// generate ModelOutput.cs
var modelOutputCSFileContent = new OnnxModelOutputClass()
{
Namespace = namespaceValue,
Target = _settings.Target,
}.TransformText();
modelOutputCSFileContent = Utils.FormatCode(modelOutputCSFileContent);

// generate ConsumeModel.cs
var consumeModelCSFileContent = new AzureAttachImageConsumeModel()
{
Namespace = namespaceValue,
Target = _settings.Target,
}.TransformText();
consumeModelCSFileContent = Utils.FormatCode(consumeModelCSFileContent);

// generate Model.csproj
var modelProjectFileContent = GenerateModelProjectFileContent(false,
false, false, true,
false, false,true, _settings.StablePackageVersion, _settings.UnstablePackageVersion);
_settings.StablePackageVersion, _settings.UnstablePackageVersion, _settings.Target);

return (modelInputCSFileContent, modelOutputCSFileContent, consumeModelCSFileContent, modelProjectFileContent);
}
Expand Down Expand Up @@ -366,7 +334,7 @@ internal IList<string> GenerateClassLabels(IDictionary<string, CodeGeneratorSett
private static string GenerateModelProjectFileContent(bool includeLightGbmPackage,
bool includeMklComponentsPackage, bool includeFastTreePackage, bool includeImageTransformerPackage,
bool includeImageClassificationPackage, bool includeRecommenderPackage, bool includeOnnxModel,
string stablePackageVersion, string unstablePackageVersion)
string stablePackageVersion, string unstablePackageVersion, GenerateTarget target)
{
ModelProject modelProject = new ModelProject()
{
Expand All @@ -378,7 +346,8 @@ private static string GenerateModelProjectFileContent(bool includeLightGbmPackag
IncludeOnnxModel = includeOnnxModel,
IncludeRecommenderPackage = includeRecommenderPackage,
StablePackageVersion = stablePackageVersion,
UnstablePackageVersion = unstablePackageVersion
UnstablePackageVersion = unstablePackageVersion,
Target = target,
};

return modelProject.TransformText();
Expand Down Expand Up @@ -411,7 +380,7 @@ private static string GeneratPredictProjectFileContent(string namespaceValue, bo
bool includeMklComponentsPackage, bool includeFastTreePackage, bool includeImageTransformerPackage,
bool includeImageClassificationPackage, bool includeRecommenderPackage,
bool includeOnnxPackage, bool includeResNet18Package,
string stablePackageVersion, string unstablePackageVersion)
string stablePackageVersion, string unstablePackageVersion, GenerateTarget target)
{
var predictProjectFileContent = new PredictProject()
{
Expand All @@ -425,7 +394,8 @@ private static string GeneratPredictProjectFileContent(string namespaceValue, bo
IncludeResNet18Package = includeResNet18Package,
IncludeRecommenderPackage = includeRecommenderPackage,
StablePackageVersion = stablePackageVersion,
UnstablePackageVersion = unstablePackageVersion
UnstablePackageVersion = unstablePackageVersion,
Target = target,
};
return predictProjectFileContent.TransformText();
}
Expand Down
9 changes: 8 additions & 1 deletion src/Microsoft.ML.CodeGenerator/Templates/Console/ModelProject.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,13 @@ public virtual string TransformText()
this.Write(" <None Update=\"bestModel.onnx\">\r\n <CopyToOutputDirectory>PreserveNewest</" +
"CopyToOutputDirectory>\r\n </None>\r\n");
}
this.Write(" </ItemGroup>\r\n \r\n</Project>\r\n");
this.Write(" </ItemGroup>\r\n\r\n <ItemGroup>\r\n");
if (Target==CSharp.GenerateTarget.Cli) {
this.Write(" <ProjectCapability Include=\"MLNETCLIGenerated\" />\r\n");
}else{
this.Write(" <ProjectCapability Include=\"ModelBuilderGenerated\" />\r\n");
}
this.Write(" </ItemGroup>\r\n</Project>\r\n");
return this.GenerationEnvironment.ToString();
}

Expand All @@ -85,6 +91,7 @@ public virtual string TransformText()
public bool IncludeRecommenderPackage {get;set;}
public string StablePackageVersion {get;set;}
public string UnstablePackageVersion {get;set;}
internal CSharp.GenerateTarget Target {get;set;}

}
#region Base class
Expand Down
10 changes: 9 additions & 1 deletion src/Microsoft.ML.CodeGenerator/Templates/Console/ModelProject.tt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,14 @@
</None>
<#}#>
</ItemGroup>


<ItemGroup>
<# if (Target==CSharp.GenerateTarget.Cli) {#>
<ProjectCapability Include="MLNETCLIGenerated" />
<#}else{#>
<ProjectCapability Include="ModelBuilderGenerated" />
<#}#>
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What's diverging between CLI and MB that we need MLNETCLIGenerated vs ModelBuilderGenerated?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's from our PM, we want to differentiate ModelBuilder-Generated from MLNet CLI Generated in telemetry data

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sounds good.

</ItemGroup>
</Project>
<#+
public bool IncludeLightGBMPackage {get;set;}
Expand All @@ -56,4 +63,5 @@ public bool IncludeOnnxModel {get; set;}
public bool IncludeRecommenderPackage {get;set;}
public string StablePackageVersion {get;set;}
public string UnstablePackageVersion {get;set;}
internal CSharp.GenerateTarget Target {get;set;}
#>
9 changes: 8 additions & 1 deletion src/Microsoft.ML.CodeGenerator/Templates/Console/PredictProject.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,13 @@ public virtual string TransformText()
this.Write(this.ToStringHelper.ToStringWithCulture(Namespace));
this.Write(".Model\\");
this.Write(this.ToStringHelper.ToStringWithCulture(Namespace));
this.Write(".Model.csproj\" />\r\n </ItemGroup>\r\n</Project>\r\n");
this.Write(".Model.csproj\" />\r\n </ItemGroup>\r\n\r\n <ItemGroup>\r\n");
if (Target==CSharp.GenerateTarget.Cli) {
this.Write(" <ProjectCapability Include=\"MLNETCLIGenerated\" />\r\n");
}else{
this.Write(" <ProjectCapability Include=\"ModelBuilderGenerated\" />\r\n");
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should modify the upstream .tt so the .cs is formatted properly here.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same w/ ModelProject.tt

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What you mean

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@LittleLittleCloud aren't these files generated from TT files?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In this PR it looks like the CS files were modified directly instead of changing the TT files.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nevermind I see them now :)

Copy link
Contributor

@justinormont justinormont Apr 6, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The newly added code is currently being formatted as:

machinelearning/src/Microsoft.ML.CodeGenerator/Templates/Console/PredictProject.cs

Lines 77 to 86 in e84006b

this.Write(this.ToStringHelper.ToStringWithCulture(Namespace));
this.Write(".Model.csproj\" />\r\n </ItemGroup>\r\n\r\n <ItemGroup>\r\n");
if (Target==CSharp.GenerateTarget.Cli) {
this.Write(" <ProjectCapability Include=\"MLNETCLIGenerated\" />\r\n");
}else{
this.Write(" <ProjectCapability Include=\"ModelBuilderGenerated\" />\r\n");
}
this.Write(" </ItemGroup>\r\n</Project>\r\n");
return this.GenerationEnvironment.ToString();
}

It should be:

            this.Write(this.ToStringHelper.ToStringWithCulture(Namespace));
            this.Write(".Model.csproj\" />\r\n  </ItemGroup>\r\n\r\n  <ItemGroup>\r\n");
            if (Target==CSharp.GenerateTarget.Cli) {
                this.Write("    <ProjectCapability Include=\"MLNETCLIGenerated\" />\r\n");
            }
            else {
                this.Write("    <ProjectCapability Include=\"ModelBuilderGenerated\" />\r\n");
            }
            this.Write("  </ItemGroup>\r\n</Project>\r\n");
            return this.GenerationEnvironment.ToString();
        }

The generated .cs files are created from their corresponding .tt T4 template files. The change would occur there. I left a comment on the .cs as it shows the output.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@JakeRadMSFT it's auto-gen, I didn't modify those cs file a bit :)

this.Write(" </ItemGroup>\r\n</Project>\r\n");
return this.GenerationEnvironment.ToString();
}

Expand All @@ -90,6 +96,7 @@ public virtual string TransformText()
public bool IncludeRecommenderPackage {get;set;}
public string StablePackageVersion {get;set;}
public string UnstablePackageVersion {get;set;}
internal CSharp.GenerateTarget Target {get;set;}

}
#region Base class
Expand Down
9 changes: 9 additions & 0 deletions src/Microsoft.ML.CodeGenerator/Templates/Console/PredictProject.tt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,14 @@
<ItemGroup>
<ProjectReference Include="..\\<#= Namespace #>.Model\\<#= Namespace #>.Model.csproj" />
</ItemGroup>

<ItemGroup>
<# if (Target==CSharp.GenerateTarget.Cli) {#>
<ProjectCapability Include="MLNETCLIGenerated" />
<#}else{#>
<ProjectCapability Include="ModelBuilderGenerated" />
<#}#>
</ItemGroup>
</Project>
<#+
public string Namespace {get;set;}
Expand All @@ -54,4 +62,5 @@ public bool IncludeResNet18Package {get; set;}
public bool IncludeRecommenderPackage {get;set;}
public string StablePackageVersion {get;set;}
public string UnstablePackageVersion {get;set;}
internal CSharp.GenerateTarget Target {get;set;}
Copy link
Contributor

@JakeRadMSFT JakeRadMSFT Apr 6, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

NIT: I would think of CLI vs ModelBuilder as the Source instead of Target. It's the thing causing the projects to get generated.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Will do

#>
4 changes: 4 additions & 0 deletions ...ests/ConsoleCodeGeneratorTests.AzureCodeGeneratorTest.test.ConsoleApp.csproj.approved.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,8 @@
<ItemGroup>
<ProjectReference Include="..\Test.Model\Test.Model.csproj" />
</ItemGroup>

<ItemGroup>
<ProjectCapability Include="ModelBuilderGenerated" />
</ItemGroup>
</Project>
5 changes: 4 additions & 1 deletion ...ovalTests/ConsoleCodeGeneratorTests.AzureCodeGeneratorTest.test.Model.csproj.approved.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,8 @@
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>


<ItemGroup>
<ProjectCapability Include="ModelBuilderGenerated" />
</ItemGroup>
</Project>
4 changes: 4 additions & 0 deletions ...CodeGeneratorTests.AzureImageCodeGeneratorTest.CodeGenTest.ConsoleApp.csproj.approved.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,8 @@
<ItemGroup>
<ProjectReference Include="..\CodeGenTest.Model\CodeGenTest.Model.csproj" />
</ItemGroup>

<ItemGroup>
<ProjectCapability Include="ModelBuilderGenerated" />
</ItemGroup>
</Project>
5 changes: 4 additions & 1 deletion ...nsoleCodeGeneratorTests.AzureImageCodeGeneratorTest.CodeGenTest.Model.csproj.approved.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,5 +17,8 @@
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>


<ItemGroup>
<ProjectCapability Include="ModelBuilderGenerated" />
</ItemGroup>
</Project>
4 changes: 4 additions & 0 deletions ...sts/ApprovalTests/ConsoleCodeGeneratorTests.ConsoleAppProjectFileContentTest.approved.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,8 @@
<ItemGroup>
<ProjectReference Include="..\MyNamespace.Model\MyNamespace.Model.csproj" />
</ItemGroup>

<ItemGroup>
<ProjectCapability Include="MLNETCLIGenerated" />
</ItemGroup>
</Project>
5 changes: 4 additions & 1 deletion ...ests/ConsoleCodeGeneratorTests.ModelProjectFileContentTestOnlyStableProjects.approved.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,5 +15,8 @@
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>


<ItemGroup>
<ProjectCapability Include="MLNETCLIGenerated" />
</ItemGroup>
</Project>
4 changes: 4 additions & 0 deletions ...rTests.Recommendation_GenerateConsoleAppProjectContents_VerifyPredictProject.approved.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,8 @@
<ItemGroup>
<ProjectReference Include="..\MyNamespace.Model\MyNamespace.Model.csproj" />
</ItemGroup>

<ItemGroup>
<ProjectCapability Include="MLNETCLIGenerated" />
</ItemGroup>
</Project>
5 changes: 4 additions & 1 deletion ...eneratorTests.Recommendation_GenerateModelProjectContents_VerifyModelProject.approved.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,5 +13,8 @@
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>


<ItemGroup>
<ProjectCapability Include="MLNETCLIGenerated" />
</ItemGroup>
</Project>