diff --git a/.gitignore b/.gitignore index 6af1d15f0a..de92124490 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,5 @@ __pycache__ build *.egg .vscode/settings.json -.ipynb_checkpoints \ No newline at end of file +.ipynb_checkpoints +.vscode/launch.json diff --git a/README.md b/README.md index 9da57aaed8..a50484a8fc 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ search = openai.Engine(id="deployment-namme").search(documents=["White House", " print(search) ``` -Please note that for the moment, the Microsoft Azure endpoints can only be used for completion and search operations. +Please note that for the moment, the Microsoft Azure endpoints can only be used for completion, search and fine-tuning operations. ### Command-line interface diff --git a/openai/api_requestor.py b/openai/api_requestor.py index ac5c5af6c6..8bb4eb6a83 100644 --- a/openai/api_requestor.py +++ b/openai/api_requestor.py @@ -122,7 +122,7 @@ def request( def handle_error_response(self, rbody, rcode, resp, rheaders, stream_error=False): try: - error_data = resp["error"] + error_data = resp["error"] if self.api_type == ApiType.OPEN_AI else resp except (KeyError, TypeError): raise error.APIError( "Invalid response object from API: %r (HTTP response code " @@ -333,6 +333,10 @@ def _interpret_response( def _interpret_response_line( self, rbody, rcode, rheaders, stream: bool ) -> OpenAIResponse: + # HTTP 204 response code does not have any content in the body. + if rcode == 204: + return OpenAIResponse(None, rheaders) + if rcode == 503: raise error.ServiceUnavailableError( "The server is overloaded or not ready yet.", diff --git a/openai/api_resources/abstract/api_resource.py b/openai/api_resources/abstract/api_resource.py index 5d10da1d29..c7245aecd4 100644 --- a/openai/api_resources/abstract/api_resource.py +++ b/openai/api_resources/abstract/api_resource.py @@ -8,7 +8,8 @@ class APIResource(OpenAIObject): api_prefix = "" - azure_api_prefix = "openai/deployments" + azure_api_prefix = "openai" + azure_deployments_prefix = "deployments" @classmethod def retrieve(cls, id, api_key=None, request_id=None, **params): @@ -46,27 +47,34 @@ def instance_url(self, operation=None): "id", ) api_version = self.api_version or openai.api_version + extn = quote_plus(id) if self.typed_api_type == ApiType.AZURE: if not api_version: raise error.InvalidRequestError( "An API version is required for the Azure API type." ) + if not operation: - raise error.InvalidRequestError( - "The request needs an operation (eg: 'search') for the Azure OpenAI API type." + base = self.class_url() + return "/%s%s/%s?api-version=%s" % ( + self.azure_api_prefix, + base, + extn, + api_version ) - extn = quote_plus(id) - return "/%s/%s/%s?api-version=%s" % ( + + return "/%s/%s/%s/%s?api-version=%s" % ( self.azure_api_prefix, + self.azure_deployments_prefix, extn, operation, - api_version, + api_version ) + elif self.typed_api_type == ApiType.OPEN_AI: base = self.class_url() - extn = quote_plus(id) return "%s/%s" % (base, extn) else: @@ -81,6 +89,7 @@ def _static_request( url_, api_key=None, api_base=None, + api_type=None, request_id=None, api_version=None, organization=None, @@ -91,6 +100,7 @@ def _static_request( api_version=api_version, organization=organization, api_base=api_base, + api_type=api_type ) response, _, api_key = requestor.request( method_, url_, params, request_id=request_id @@ -98,3 +108,10 @@ def _static_request( return util.convert_to_openai_object( response, api_key, api_version, organization ) + + @classmethod + def _get_api_type_and_version(cls, api_type: str, api_version: str): + typed_api_type = ApiType.from_str(api_type) if api_type else ApiType.from_str(openai.api_type) + typed_api_version = api_version or openai.api_version + return (typed_api_type, typed_api_version) + diff --git a/openai/api_resources/abstract/createable_api_resource.py b/openai/api_resources/abstract/createable_api_resource.py index 1538cb3719..6ca2368d13 100644 --- a/openai/api_resources/abstract/createable_api_resource.py +++ b/openai/api_resources/abstract/createable_api_resource.py @@ -1,5 +1,6 @@ -from openai import api_requestor, util +from openai import api_requestor, util, error from openai.api_resources.abstract.api_resource import APIResource +from openai.util import ApiType class CreateableAPIResource(APIResource): @@ -10,6 +11,7 @@ def create( cls, api_key=None, api_base=None, + api_type=None, request_id=None, api_version=None, organization=None, @@ -18,10 +20,20 @@ def create( requestor = api_requestor.APIRequestor( api_key, api_base=api_base, + api_type=api_type, api_version=api_version, organization=organization, ) - url = cls.class_url() + typed_api_type, api_version = cls._get_api_type_and_version(api_type, api_version) + + if typed_api_type == ApiType.AZURE: + base = cls.class_url() + url = "/%s%s?api-version=%s" % (cls.azure_api_prefix, base, api_version) + elif typed_api_type == ApiType.OPEN_AI: + url = cls.class_url() + else: + raise error.InvalidAPIType('Unsupported API type %s' % api_type) + response, _, api_key = requestor.request( "post", url, params, request_id=request_id ) diff --git a/openai/api_resources/abstract/deletable_api_resource.py b/openai/api_resources/abstract/deletable_api_resource.py index 47111b153c..3a6e83ff0e 100644 --- a/openai/api_resources/abstract/deletable_api_resource.py +++ b/openai/api_resources/abstract/deletable_api_resource.py @@ -1,12 +1,24 @@ from urllib.parse import quote_plus +from openai import error from openai.api_resources.abstract.api_resource import APIResource - +from openai.util import ApiType class DeletableAPIResource(APIResource): @classmethod - def delete(cls, sid, **params): + def delete(cls, sid, api_type=None, api_version=None, **params): if isinstance(cls, APIResource): raise ValueError(".delete may only be called as a class method now.") - url = "%s/%s" % (cls.class_url(), quote_plus(sid)) - return cls._static_request("delete", url, **params) + + base = cls.class_url() + extn = quote_plus(sid) + + typed_api_type, api_version = cls._get_api_type_and_version(api_type, api_version) + if typed_api_type == ApiType.AZURE: + url = "/%s%s/%s?api-version=%s" % (cls.azure_api_prefix, base, extn, api_version) + elif typed_api_type == ApiType.OPEN_AI: + url = "%s/%s" % (base, extn) + else: + raise error.InvalidAPIType('Unsupported API type %s' % api_type) + + return cls._static_request("delete", url, api_type=api_type, api_version=api_version, **params) diff --git a/openai/api_resources/abstract/engine_api_resource.py b/openai/api_resources/abstract/engine_api_resource.py index cd79528df4..b07ab2ec02 100644 --- a/openai/api_resources/abstract/engine_api_resource.py +++ b/openai/api_resources/abstract/engine_api_resource.py @@ -15,7 +15,6 @@ class EngineAPIResource(APIResource): engine_required = True plain_old_data = False - azure_api_prefix = "openai/deployments" def __init__(self, engine: Optional[str] = None, **kwargs): super().__init__(engine=engine, **kwargs) @@ -30,12 +29,7 @@ def class_url( # Namespaces are separated in object names with periods (.) and in URLs # with forward slashes (/), so replace the former with the latter. base = cls.OBJECT_NAME.replace(".", "/") # type: ignore - typed_api_type = ( - ApiType.from_str(api_type) - if api_type - else ApiType.from_str(openai.api_type) - ) - api_version = api_version or openai.api_version + typed_api_type, api_version = cls._get_api_type_and_version(api_type, api_version) if typed_api_type == ApiType.AZURE: if not api_version: @@ -47,11 +41,12 @@ def class_url( "You must provide the deployment name in the 'engine' parameter to access the Azure OpenAI service" ) extn = quote_plus(engine) - return "/%s/%s/%ss?api-version=%s" % ( + return "/%s/%s/%s/%ss?api-version=%s" % ( cls.azure_api_prefix, + cls.azure_deployments_prefix, extn, base, - api_version, + api_version ) elif typed_api_type == ApiType.OPEN_AI: @@ -148,27 +143,29 @@ def instance_url(self): "id", ) - params_connector = "?" + extn = quote_plus(id) + params_connector = '?' + if self.typed_api_type == ApiType.AZURE: api_version = self.api_version or openai.api_version if not api_version: raise error.InvalidRequestError( "An API version is required for the Azure API type." ) - extn = quote_plus(id) base = self.OBJECT_NAME.replace(".", "/") - url = "/%s/%s/%ss/%s?api-version=%s" % ( + url = "/%s/%s/%s/%ss/%s?api-version=%s" % ( self.azure_api_prefix, + self.azure_deployments_prefix, self.engine, base, extn, - api_version, + api_version ) - params_connector = "&" + params_connector = '&' + elif self.typed_api_type == ApiType.OPEN_AI: base = self.class_url(self.engine, self.api_type, self.api_version) - extn = quote_plus(id) url = "%s/%s" % (base, extn) else: diff --git a/openai/api_resources/abstract/listable_api_resource.py b/openai/api_resources/abstract/listable_api_resource.py index b9cf952a91..c01af74236 100644 --- a/openai/api_resources/abstract/listable_api_resource.py +++ b/openai/api_resources/abstract/listable_api_resource.py @@ -1,5 +1,6 @@ -from openai import api_requestor, util +from openai import api_requestor, util, error from openai.api_resources.abstract.api_resource import APIResource +from openai.util import ApiType class ListableAPIResource(APIResource): @@ -15,15 +16,27 @@ def list( api_version=None, organization=None, api_base=None, + api_type=None, **params, ): requestor = api_requestor.APIRequestor( api_key, api_base=api_base or cls.api_base(), api_version=api_version, + api_type=api_type, organization=organization, ) - url = cls.class_url() + + typed_api_type, api_version = cls._get_api_type_and_version(api_type, api_version) + + if typed_api_type == ApiType.AZURE: + base = cls.class_url() + url = "/%s%s?api-version=%s" % (cls.azure_api_prefix, base, api_version) + elif typed_api_type == ApiType.OPEN_AI: + url = cls.class_url() + else: + raise error.InvalidAPIType('Unsupported API type %s' % api_type) + response, _, api_key = requestor.request( "get", url, params, request_id=request_id ) diff --git a/openai/api_resources/file.py b/openai/api_resources/file.py index dbe387b157..2780f543d4 100644 --- a/openai/api_resources/file.py +++ b/openai/api_resources/file.py @@ -3,8 +3,9 @@ from typing import cast import openai -from openai import api_requestor, util +from openai import api_requestor, util, error from openai.api_resources.abstract import DeletableAPIResource, ListableAPIResource +from openai.util import ApiType class File(ListableAPIResource, DeletableAPIResource): @@ -18,6 +19,7 @@ def create( model=None, api_key=None, api_base=None, + api_type=None, api_version=None, organization=None, user_provided_filename=None, @@ -27,19 +29,29 @@ def create( requestor = api_requestor.APIRequestor( api_key, api_base=api_base or openai.api_base, + api_type=api_type, api_version=api_version, organization=organization, ) - url = cls.class_url() + typed_api_type, api_version = cls._get_api_type_and_version(api_type, api_version) + + if typed_api_type == ApiType.AZURE: + base = cls.class_url() + url = "/%s%s?api-version=%s" % (cls.azure_api_prefix, base, api_version) + elif typed_api_type == ApiType.OPEN_AI: + url = cls.class_url() + else: + raise error.InvalidAPIType('Unsupported API type %s' % api_type) + # Set the filename on 'purpose' and 'model' to None so they are # interpreted as form data. files = [("purpose", (None, purpose))] if model is not None: files.append(("model", (None, model))) if user_provided_filename is not None: - files.append(("file", (user_provided_filename, file))) + files.append(("file", (user_provided_filename, file, 'application/octet-stream'))) else: - files.append(("file", file)) + files.append(("file", ("file", file, 'application/octet-stream'))) response, _, api_key = requestor.request("post", url, files=files) return util.convert_to_openai_object( response, api_key, api_version, organization @@ -47,15 +59,31 @@ def create( @classmethod def download( - cls, id, api_key=None, api_base=None, api_version=None, organization=None + cls, + id, + api_key=None, + api_base=None, + api_type=None, + api_version=None, + organization=None ): requestor = api_requestor.APIRequestor( api_key, api_base=api_base or openai.api_base, + api_type=api_type, api_version=api_version, organization=organization, ) - url = f"{cls.class_url()}/{id}/content" + typed_api_type, api_version = cls._get_api_type_and_version(api_type, api_version) + + if typed_api_type == ApiType.AZURE: + base = cls.class_url() + url = "/%s%s/%s/content?api-version=%s" % (cls.azure_api_prefix, base, id, api_version) + elif typed_api_type == ApiType.OPEN_AI: + url = f"{cls.class_url()}/{id}/content" + else: + raise error.InvalidAPIType('Unsupported API type %s' % api_type) + result = requestor.request_raw("get", url) if not 200 <= result.status_code < 300: raise requestor.handle_error_response( @@ -75,6 +103,7 @@ def find_matching_files( purpose, api_key=None, api_base=None, + api_type=None, api_version=None, organization=None, ): @@ -82,6 +111,7 @@ def find_matching_files( all_files = cls.list( api_key=api_key, api_base=api_base or openai.api_base, + api_type=api_type, api_version=api_version, organization=organization, ).get("data", []) @@ -93,7 +123,9 @@ def find_matching_files( file_basename = os.path.basename(f["filename"]) if file_basename != basename: continue - if f["bytes"] != bytes: + if "bytes" in f and f["bytes"] != bytes: + continue + if "size" in f and int(f["size"]) != bytes: continue matching_files.append(f) return matching_files diff --git a/openai/api_resources/fine_tune.py b/openai/api_resources/fine_tune.py index c53671ae68..3e596b6bfd 100644 --- a/openai/api_resources/fine_tune.py +++ b/openai/api_resources/fine_tune.py @@ -1,23 +1,41 @@ from urllib.parse import quote_plus -from openai import api_requestor, util +from openai import api_requestor, util, error from openai.api_resources.abstract import ( CreateableAPIResource, ListableAPIResource, nested_resource_class_methods, ) +from openai.api_resources.abstract.deletable_api_resource import DeletableAPIResource from openai.openai_response import OpenAIResponse +from openai.util import ApiType @nested_resource_class_methods("event", operations=["list"]) -class FineTune(ListableAPIResource, CreateableAPIResource): +class FineTune(ListableAPIResource, CreateableAPIResource, DeletableAPIResource): OBJECT_NAME = "fine-tune" @classmethod - def cancel(cls, id, api_key=None, request_id=None, **params): + def cancel( + cls, + id, + api_key=None, + api_type=None, + request_id=None, + api_version=None, + **params + ): base = cls.class_url() extn = quote_plus(id) - url = "%s/%s/cancel" % (base, extn) + + typed_api_type, api_version = cls._get_api_type_and_version(api_type, api_version) + if typed_api_type == ApiType.AZURE: + url = "/%s%s/%s/cancel?api-version=%s" % (cls.azure_api_prefix, base, extn, api_version) + elif typed_api_type == ApiType.OPEN_AI: + url = "%s/%s/cancel" % (base, extn) + else: + raise error.InvalidAPIType('Unsupported API type %s' % api_type) + instance = cls(id, api_key, **params) return instance.request("post", url, request_id=request_id) @@ -27,6 +45,7 @@ def stream_events( id, api_key=None, api_base=None, + api_type=None, request_id=None, api_version=None, organization=None, @@ -38,10 +57,20 @@ def stream_events( requestor = api_requestor.APIRequestor( api_key, api_base=api_base, + api_type=api_type, api_version=api_version, organization=organization, ) - url = "%s/%s/events?stream=true" % (base, extn) + + typed_api_type, api_version = cls._get_api_type_and_version(api_type, api_version) + + if typed_api_type == ApiType.AZURE: + url = "/%s%s/%s/events?stream=true&api-version=%s" % (cls.azure_api_prefix, base, extn, api_version) + elif typed_api_type == ApiType.OPEN_AI: + url = "%s/%s/events?stream=true" % (base, extn) + else: + raise error.InvalidAPIType('Unsupported API type %s' % api_type) + response, _, api_key = requestor.request( "get", url, params, stream=True, request_id=request_id ) diff --git a/openai/cli.py b/openai/cli.py index 3b0625611f..dc8436701d 100644 --- a/openai/cli.py +++ b/openai/cli.py @@ -320,7 +320,7 @@ def _maybe_upload_file( sys.stdout.write( "Found potentially duplicated files with name '{name}', purpose 'fine-tune' and size {size} bytes\n".format( name=os.path.basename(matching_files[0]["filename"]), - size=matching_files[0]["bytes"], + size=matching_files[0]["bytes"] if "bytes" in matching_files[0] else matching_files[0]["size"], ) ) sys.stdout.write("\n".join(file_ids)) diff --git a/openai/tests/test_url_composition.py b/openai/tests/test_url_composition.py index 2b561c6d53..f5a3251dba 100644 --- a/openai/tests/test_url_composition.py +++ b/openai/tests/test_url_composition.py @@ -133,9 +133,7 @@ def test_engine_search_url_composition_azure_no_operation() -> None: engine = Engine(id="test_id", api_type="azure", api_version="2021-11-01-preview") assert engine.api_type == "azure" assert engine.typed_api_type == ApiType.AZURE - with pytest.raises(Exception): - engine.instance_url() - + assert engine.instance_url() == "/openai/engines/test_id?api-version=2021-11-01-preview" @pytest.mark.url def test_engine_search_url_composition_default() -> None: diff --git a/openai/version.py b/openai/version.py index 99977b4919..b87ca6327d 100644 --- a/openai/version.py +++ b/openai/version.py @@ -1 +1 @@ -VERSION = "0.16.0" +VERSION = "0.17.0"