Skip to content

Image generation/editing using Gemini 2.0 Flash #12

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Mar 16, 2025
Merged

Image generation/editing using Gemini 2.0 Flash #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bdd3554
Formatting fixes
swissspidy Mar 15, 2025
10d9771
Gemini 2.0 flash
swissspidy Mar 15, 2025
56ae952
POC to pass an existing image to Gemini
swissspidy Mar 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
184 changes: 95 additions & 89 deletions src/AiCommand.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,19 +45,19 @@ class AiCommand extends WP_CLI_Command {
*/
public function __invoke( $args, $assoc_args ) {
$server = new MCP\Server();
$client = new MCP\Client($server);
$client = new MCP\Client( $server );

$this->register_tools($server, $client);
$this->register_tools( $server, $client );

$this->register_resources($server);
$this->register_resources( $server );

$result = $client->call_ai_service_with_prompt( $args[0] );

WP_CLI::success( $result );
}

// Register tools for AI processing
private function register_tools($server, $client) {
private function register_tools( $server, $client ) {
$server->register_tool(
[
'name' => 'calculate_total',
Expand Down Expand Up @@ -105,135 +105,141 @@ private function register_tools($server, $client) {
]
);

// $server->register_tool(
// [
// 'name' => 'generate_image',
// 'description' => 'Generates an image.',
// 'inputSchema' => [
// 'type' => 'object',
// 'properties' => [
// 'prompt' => [
// 'type' => 'string',
// 'description' => 'The prompt for generating the image.',
// ],
// ],
// 'required' => [ 'prompt' ],
// ],
// 'callable' => function ( $params ) use ( $client ) {
// return $client->get_image_from_ai_service( $params['prompt'] );
// },
// ]
// );

$server->register_tool(
[
'name' => 'generate_image',
'description' => 'Generates an image.',
'name' => 'fetch_wp_community_events',
'description' => 'Fetches upcoming WordPress community events near a specified city or the user\'s current location. If no events are found in the exact location, nearby events within a specific radius will be considered.',
'inputSchema' => [
'type' => 'object',
'properties' => [
'prompt' => [
'location' => [
'type' => 'string',
'description' => 'The prompt for generating the image.',
'description' => 'City name or "near me" for auto-detected location. If no events are found in the exact location, the tool will also consider nearby events within a specified radius (default: 100 km).',
],
],
'required' => [ 'prompt' ],
'required' => [ 'location' ], // We only require the location
],
'callable' => function ( $params ) use ( $client ) {
return $client->get_image_from_ai_service( $params['prompt'] );
},
]
);
'callable' => function ( $params ) {
// Default user ID is 0
$user_id = 0;

// Get the location from the parameters (already supplied in the prompt)
$location_input = strtolower( trim( $params['location'] ) );

// Manually include the WP_Community_Events class if it's not loaded
if ( ! class_exists( 'WP_Community_Events' ) ) {
require_once ABSPATH . 'wp-admin/includes/class-wp-community-events.php';
}

// Determine location for the WP_Community_Events class
$location = null;
if ( $location_input !== 'near me' ) {
// Provide city name (WP will resolve coordinates)
$location = [
'description' => $location_input,
];
}

$server->register_tool(
[
'name' => 'fetch_wp_community_events',
'description' => 'Fetches upcoming WordPress community events near a specified city or the user\'s current location. If no events are found in the exact location, nearby events within a specific radius will be considered.',
'inputSchema' => [
'type' => 'object',
'properties' => [
'location' => [
'type' => 'string',
'description' => 'City name or "near me" for auto-detected location. If no events are found in the exact location, the tool will also consider nearby events within a specified radius (default: 100 km).',
],
],
'required' => [ 'location' ], // We only require the location
],
'callable' => function ( $params ) {
// Default user ID is 0
$user_id = 0;

// Get the location from the parameters (already supplied in the prompt)
$location_input = strtolower( trim( $params['location'] ) );

// Manually include the WP_Community_Events class if it's not loaded
if ( ! class_exists( 'WP_Community_Events' ) ) {
require_once ABSPATH . 'wp-admin/includes/class-wp-community-events.php';
}

// Determine location for the WP_Community_Events class
$location = null;
if ( $location_input !== 'near me' ) {
// Provide city name (WP will resolve coordinates)
$location = [
'description' => $location_input,
];
}

// Instantiate WP_Community_Events with user ID (0) and optional location
$events_instance = new WP_Community_Events( $user_id, $location );

// Get events from WP_Community_Events
$events = $events_instance->get_events($location_input);

// Check for WP_Error
if ( is_wp_error( $events ) ) {
return [ 'error' => $events->get_error_message() ];
}
// Instantiate WP_Community_Events with user ID (0) and optional location
$events_instance = new WP_Community_Events( $user_id, $location );

// Get events from WP_Community_Events
$events = $events_instance->get_events( $location_input );

// Check for WP_Error
if ( is_wp_error( $events ) ) {
return [ 'error' => $events->get_error_message() ];
}

// If no events found
if ( empty( $events['events'] ) ) {
return [ 'message' => 'No events found near ' . ( $location_input === 'near me' ? 'your location' : $location_input ) ];
}
if ( empty( $events['events'] ) ) {
return [ 'message' => 'No events found near ' . ( $location_input === 'near me' ? 'your location' : $location_input ) ];
}

// Format and return the events correctly
$formatted_events = array_map( function ( $event ) {
// Log event details to ensure properties are accessible
error_log( 'Event details: ' . print_r( $event, true ) );
$formatted_events = array_map(
function ( $event ) {
// Log event details to ensure properties are accessible
error_log( 'Event details: ' . print_r( $event, true ) );

// Initialize a formatted event string
$formatted_event = '';
// Initialize a formatted event string
$formatted_event = '';

// Format event title
if ( isset( $event['title'] ) ) {
// Format event title
if ( isset( $event['title'] ) ) {
$formatted_event .= $event['title'] . "\n";
}
}

// Format the date nicely
$formatted_event .= ' - Date: ' . ( isset( $event['date'] ) ? date( 'F j, Y g:i A', strtotime( $event['date'] ) ) : 'No date available' ) . "\n";
// Format the date nicely
$formatted_event .= ' - Date: ' . ( isset( $event['date'] ) ? date( 'F j, Y g:i A', strtotime( $event['date'] ) ) : 'No date available' ) . "\n";

// Format the location
if ( isset( $event['location']['location'] ) ) {
$formatted_event .= ' - Location: ' . $event['location']['location'] . "\n";
}
// Format the location
if ( isset( $event['location']['location'] ) ) {
$formatted_event .= ' - Location: ' . $event['location']['location'] . "\n";
}

// Format the event URL
$formatted_event .= isset( $event['url'] ) ? ' - URL: ' . $event['url'] . "\n" : '';
// Format the event URL
$formatted_event .= isset( $event['url'] ) ? ' - URL: ' . $event['url'] . "\n" : '';

return $formatted_event;
}, $events['events'] );
return $formatted_event;
},
$events['events']
);

// Combine the formatted events into a single string
$formatted_events_output = implode("\n", $formatted_events);
$formatted_events_output = implode( "\n", $formatted_events );

// Return the formatted events string
return [
'message' => "OK. I found " . count($formatted_events) . " WordPress events near " . ( $location_input === 'near me' ? 'your location' : $location_input ) . ":\n\n" . $formatted_events_output
'message' => 'OK. I found ' . count( $formatted_events ) . ' WordPress events near ' . ( $location_input === 'near me' ? 'your location' : $location_input ) . ":\n\n" . $formatted_events_output,
];
},
},
]
);

}

// Register resources for AI access
private function register_resources($server) {
private function register_resources( $server ) {
// Register Users resource
$server->register_resource([
$server->register_resource(
[
'name' => 'users',
'uri' => 'data://users',
'description' => 'List of users',
'mimeType' => 'application/json',
'dataKey' => 'users', // Data will be fetched from 'users'
]);
]
);

// Register Product Catalog resource
$server->register_resource([
$server->register_resource(
[
'name' => 'product_catalog',
'uri' => 'file://./products.json',
'description' => 'Product catalog',
'mimeType' => 'application/json',
'filePath' => './products.json', // Data will be fetched from products.json
]);
]
);
}
}
85 changes: 73 additions & 12 deletions src/MCP/Client.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,14 @@
use Felix_Arntz\AI_Services\Services\API\Enums\AI_Capability;
use Felix_Arntz\AI_Services\Services\API\Enums\Content_Role;
use Felix_Arntz\AI_Services\Services\API\Helpers;
use Felix_Arntz\AI_Services\Services\API\Types\Blob;
use Felix_Arntz\AI_Services\Services\API\Types\Content;
use Felix_Arntz\AI_Services\Services\API\Types\Parts;
use Felix_Arntz\AI_Services\Services\API\Types\Parts\File_Data_Part;
use Felix_Arntz\AI_Services\Services\API\Types\Parts\Function_Call_Part;
use Felix_Arntz\AI_Services\Services\API\Types\Parts\Inline_Data_Part;
use Felix_Arntz\AI_Services\Services\API\Types\Parts\Text_Part;
use Felix_Arntz\AI_Services\Services\API\Types\Text_Generation_Config;
use Felix_Arntz\AI_Services\Services\API\Types\Tools;
use WP_CLI;

Expand Down Expand Up @@ -130,9 +132,20 @@ static function () {
public function call_ai_service_with_prompt( string $prompt ) {
$parts = new Parts();
$parts->add_text_part( $prompt );
$content = new Content( Content_Role::USER, $parts );

return $this->call_ai_service( [ $content ] );
$contents = [
new Content( Content_Role::USER, $parts ),
];

// $parts = new Parts();
// $parts->add_inline_data_part(
// 'image/png',
// Helpers::blob_to_base64_data_url( new Blob( file_get_contents( '/private/tmp/ai-generated-imaget1sjmomi30i31C1YtZy.png' ), 'image/png' ) ),
// );
//
// $contents[] = $parts;

return $this->call_ai_service( $contents );
}

private function call_ai_service( $contents ) {
Expand Down Expand Up @@ -172,25 +185,33 @@ static function () {
]
);

\WP_CLI::debug( 'Making request...' . print_r( $contents, true ), 'ai' );
\WP_CLI::debug( 'Making request...' . print_r( $contents, true ), 'ai' );

if ( $service->get_service_slug() === 'openai' ) {
$model = 'gpt-4o';
} else {
$model = 'gemini-2.0-flash';
$model = 'gemini-2.0-flash-exp';
}

$candidates = $service
->get_model(
[
'feature' => 'text-generation',
'model' => $model,
'tools' => $tools,
'capabilities' => [
AI_Capability::MULTIMODAL_INPUT,
AI_Capability::TEXT_GENERATION,
AI_Capability::FUNCTION_CALLING,
],
'feature' => 'text-generation',
'model' => $model,
// 'tools' => $tools,
'capabilities' => [
AI_Capability::MULTIMODAL_INPUT,
AI_Capability::TEXT_GENERATION,
// AI_Capability::FUNCTION_CALLING,
],
'generationConfig' => Text_Generation_Config::from_array(
array(
'responseModalities' => array(
'Text',
'Image',
),
)
),
]
)
->generate_text( $contents );
Expand Down Expand Up @@ -224,6 +245,46 @@ static function () {
$parts->add_function_response_part( $part->get_id(), $part->get_name(), $function_result );
$content = new Content( Content_Role::USER, $parts );
$new_contents[] = $content;
} elseif ( $part instanceof Inline_Data_Part ) {
$image_url = $part->get_base64_data(); // Data URL.
$image_blob = Helpers::base64_data_url_to_blob( $image_url );

if ( $image_blob ) {
$filename = tempnam( '/tmp', 'ai-generated-image' );
$parts = explode( '/', $part->get_mime_type() );
$extension = $parts[1];
rename( $filename, $filename . '.' . $extension );
$filename .= '.' . $extension;

file_put_contents( $filename, $image_blob->get_binary_data() );

$image_url = $filename;
} else {
$binary_data = base64_decode( $image_url );
if ( false !== $binary_data ) {
$image_blob = new Blob( $binary_data, $part->get_mime_type() );

$filename = tempnam( '/tmp', 'ai-generated-image' );
$parts = explode( '/', $part->get_mime_type() );
$extension = $parts[1];
rename( $filename, $filename . '.' . $extension );
$filename .= '.' . $extension;

file_put_contents( $filename, $image_blob->get_binary_data() );

$image_url = $filename;
}
}

$text .= "Generated image: $image_url\n";

break;
}

if ( $part instanceof File_Data_Part ) {
$image_url = $part->get_file_uri(); // Actual URL. May have limited TTL (often 1 hour).
// TODO: Save as file or so.
break;
}
}

Expand Down