diff --git a/mlir/include/mlir/Dialect/ArmSVE/CMakeLists.txt b/mlir/include/mlir/Dialect/ArmSVE/CMakeLists.txt
index 06595b7088a1e..f33061b2d87cf 100644
--- a/mlir/include/mlir/Dialect/ArmSVE/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/ArmSVE/CMakeLists.txt
@@ -1,6 +1 @@
-add_mlir_dialect(ArmSVE arm_sve ArmSVE)
-add_mlir_doc(ArmSVE ArmSVE Dialects/ -gen-dialect-doc -dialect=arm_sve)
-
-set(LLVM_TARGET_DEFINITIONS ArmSVE.td)
-mlir_tablegen(ArmSVEConversions.inc -gen-llvmir-conversions)
-add_public_tablegen_target(MLIRArmSVEConversionsIncGen)
+add_subdirectory(IR)
diff --git a/mlir/include/mlir/Dialect/ArmSVE/ArmSVE.td b/mlir/include/mlir/Dialect/ArmSVE/IR/ArmSVE.td
similarity index 96%
rename from mlir/include/mlir/Dialect/ArmSVE/ArmSVE.td
rename to mlir/include/mlir/Dialect/ArmSVE/IR/ArmSVE.td
index 5c86df1ef21f4..58dec6091f27f 100644
--- a/mlir/include/mlir/Dialect/ArmSVE/ArmSVE.td
+++ b/mlir/include/mlir/Dialect/ArmSVE/IR/ArmSVE.td
@@ -28,7 +28,6 @@ def ArmSVE_Dialect : Dialect {
     This dialect contains the definitions necessary to target specific Arm SVE
     scalable vector operations.
   }];
-
 }
 
 //===----------------------------------------------------------------------===//
@@ -38,16 +37,23 @@ def ArmSVE_Dialect : Dialect {
 class ArmSVE_Op<string mnemonic, list<Trait> traits = []> :
   Op<ArmSVE_Dialect, mnemonic, traits> {}
 
-class ArmSVE_IntrBinaryOverloadedOp<string mnemonic,
-                                    list<Trait> traits = []> :
+class ArmSVE_IntrOp<string mnemonic,
+                    list<Trait> traits = [],
+                    list<int> overloadedOperands = [],
+                    list<int> overloadedResults = []> :
   LLVM_IntrOpBase</*Dialect dialect=*/ArmSVE_Dialect,
                   /*string opName=*/"intr." # mnemonic,
                   /*string enumName=*/"aarch64_sve_" # !subst(".", "_", mnemonic),
-                  /*list<int> overloadedResults=*/[0],
-                  /*list<int> overloadedOperands=*/[], // defined by result overload
+                  /*list<int> overloadedResults=*/overloadedResults,
+                  /*list<int> overloadedOperands=*/overloadedOperands,
                   /*list<Trait> traits=*/traits,
                   /*int numResults=*/1>;
 
+class ArmSVE_IntrBinaryOverloadedOp<string mnemonic,
+                                    list<Trait> traits = []>:
+  ArmSVE_IntrOp<mnemonic, traits,
+    /*overloadedOperands=*/[], /*overloadedResults=*/[0]>;
+
 class ScalableMaskedFOp<string mnemonic, string op_description,
                         list<Trait> traits = []> :
   ArmSVE_Op<mnemonic, !listconcat(traits,
diff --git a/mlir/include/mlir/Dialect/ArmSVE/ArmSVEDialect.h b/mlir/include/mlir/Dialect/ArmSVE/IR/ArmSVEDialect.h
similarity index 89%
rename from mlir/include/mlir/Dialect/ArmSVE/ArmSVEDialect.h
rename to mlir/include/mlir/Dialect/ArmSVE/IR/ArmSVEDialect.h
index 6ecee1e4552b9..a424e109b06d3 100644
--- a/mlir/include/mlir/Dialect/ArmSVE/ArmSVEDialect.h
+++ b/mlir/include/mlir/Dialect/ArmSVE/IR/ArmSVEDialect.h
@@ -19,9 +19,9 @@
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 
-#include "mlir/Dialect/ArmSVE/ArmSVEDialect.h.inc"
+#include "mlir/Dialect/ArmSVE/IR/ArmSVEDialect.h.inc"
 
 #define GET_OP_CLASSES
-#include "mlir/Dialect/ArmSVE/ArmSVE.h.inc"
+#include "mlir/Dialect/ArmSVE/IR/ArmSVE.h.inc"
 
 #endif // MLIR_DIALECT_ARMSVE_ARMSVEDIALECT_H
diff --git a/mlir/include/mlir/Dialect/ArmSVE/IR/CMakeLists.txt b/mlir/include/mlir/Dialect/ArmSVE/IR/CMakeLists.txt
new file mode 100644
index 0000000000000..06595b7088a1e
--- /dev/null
+++ b/mlir/include/mlir/Dialect/ArmSVE/IR/CMakeLists.txt
@@ -0,0 +1,6 @@
+add_mlir_dialect(ArmSVE arm_sve ArmSVE)
+add_mlir_doc(ArmSVE ArmSVE Dialects/ -gen-dialect-doc -dialect=arm_sve)
+
+set(LLVM_TARGET_DEFINITIONS ArmSVE.td)
+mlir_tablegen(ArmSVEConversions.inc -gen-llvmir-conversions)
+add_public_tablegen_target(MLIRArmSVEConversionsIncGen)
diff --git a/mlir/include/mlir/Dialect/ArmSVE/Transforms.h b/mlir/include/mlir/Dialect/ArmSVE/Transforms/Transforms.h
similarity index 100%
rename from mlir/include/mlir/Dialect/ArmSVE/Transforms.h
rename to mlir/include/mlir/Dialect/ArmSVE/Transforms/Transforms.h
diff --git a/mlir/include/mlir/InitAllDialects.h b/mlir/include/mlir/InitAllDialects.h
index 5ec36a7f289e5..d04ed373ecf04 100644
--- a/mlir/include/mlir/InitAllDialects.h
+++ b/mlir/include/mlir/InitAllDialects.h
@@ -24,7 +24,7 @@
 #include "mlir/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.h"
 #include "mlir/Dialect/ArmNeon/ArmNeonDialect.h"
 #include "mlir/Dialect/ArmSME/IR/ArmSME.h"
-#include "mlir/Dialect/ArmSVE/ArmSVEDialect.h"
+#include "mlir/Dialect/ArmSVE/IR/ArmSVEDialect.h"
 #include "mlir/Dialect/Async/IR/Async.h"
 #include "mlir/Dialect/Bufferization/IR/Bufferization.h"
 #include "mlir/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.h"
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp
index 2929823bad32a..b865a2671fff7 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp
@@ -17,8 +17,8 @@
 #include "mlir/Dialect/ArmSME/IR/ArmSME.h"
 #include "mlir/Dialect/ArmSME/Transforms/Passes.h"
 #include "mlir/Dialect/ArmSME/Transforms/Transforms.h"
-#include "mlir/Dialect/ArmSVE/ArmSVEDialect.h"
-#include "mlir/Dialect/ArmSVE/Transforms.h"
+#include "mlir/Dialect/ArmSVE/IR/ArmSVEDialect.h"
+#include "mlir/Dialect/ArmSVE/Transforms/Transforms.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
diff --git a/mlir/lib/Dialect/ArmSVE/IR/ArmSVEDialect.cpp b/mlir/lib/Dialect/ArmSVE/IR/ArmSVEDialect.cpp
index 4af836a93c2a1..b7f1020deba1e 100644
--- a/mlir/lib/Dialect/ArmSVE/IR/ArmSVEDialect.cpp
+++ b/mlir/lib/Dialect/ArmSVE/IR/ArmSVEDialect.cpp
@@ -10,7 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Dialect/ArmSVE/ArmSVEDialect.h"
+#include "mlir/Dialect/ArmSVE/IR/ArmSVEDialect.h"
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/DialectImplementation.h"
@@ -38,17 +38,17 @@ static Type getI1SameShape(Type type) {
 // Tablegen Definitions
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Dialect/ArmSVE/ArmSVEDialect.cpp.inc"
+#include "mlir/Dialect/ArmSVE/IR/ArmSVEDialect.cpp.inc"
 
 #define GET_OP_CLASSES
-#include "mlir/Dialect/ArmSVE/ArmSVE.cpp.inc"
+#include "mlir/Dialect/ArmSVE/IR/ArmSVE.cpp.inc"
 
 #define GET_TYPEDEF_CLASSES
-#include "mlir/Dialect/ArmSVE/ArmSVETypes.cpp.inc"
+#include "mlir/Dialect/ArmSVE/IR/ArmSVETypes.cpp.inc"
 
 void ArmSVEDialect::initialize() {
   addOperations<
 #define GET_OP_LIST
-#include "mlir/Dialect/ArmSVE/ArmSVE.cpp.inc"
+#include "mlir/Dialect/ArmSVE/IR/ArmSVE.cpp.inc"
       >();
 }
diff --git a/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp b/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp
index b6723a52e177f..abbb978304068 100644
--- a/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp
+++ b/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp
@@ -8,8 +8,8 @@
 
 #include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
 #include "mlir/Conversion/LLVMCommon/Pattern.h"
-#include "mlir/Dialect/ArmSVE/ArmSVEDialect.h"
-#include "mlir/Dialect/ArmSVE/Transforms.h"
+#include "mlir/Dialect/ArmSVE/IR/ArmSVEDialect.h"
+#include "mlir/Dialect/ArmSVE/Transforms/Transforms.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/IR/BuiltinOps.h"
diff --git a/mlir/lib/Target/LLVMIR/Dialect/ArmSVE/ArmSVEToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/ArmSVE/ArmSVEToLLVMIRTranslation.cpp
index bc1f0e934fa02..cd10811b68f02 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/ArmSVE/ArmSVEToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/ArmSVE/ArmSVEToLLVMIRTranslation.cpp
@@ -11,7 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Target/LLVMIR/Dialect/ArmSVE/ArmSVEToLLVMIRTranslation.h"
-#include "mlir/Dialect/ArmSVE/ArmSVEDialect.h"
+#include "mlir/Dialect/ArmSVE/IR/ArmSVEDialect.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/Target/LLVMIR/ModuleTranslation.h"
 
@@ -35,7 +35,7 @@ class ArmSVEDialectLLVMIRTranslationInterface
   convertOperation(Operation *op, llvm::IRBuilderBase &builder,
                    LLVM::ModuleTranslation &moduleTranslation) const final {
     Operation &opInst = *op;
-#include "mlir/Dialect/ArmSVE/ArmSVEConversions.inc"
+#include "mlir/Dialect/ArmSVE/IR/ArmSVEConversions.inc"
 
     return failure();
   }