- SAP Managed Tags:
What is the correct way to provision a BTP destination for calling LLMs from SAP CAP applications on BTP to GenAI Hub viaSAP AI Core?
I'm using SAP CAP plugin
npm install cap-llm-plugin
Updating package.json in my CAP project with following:
"cds": {
"requires": {
"destinations": true,
"cap-llm-plugin": true,
"gen-ai-hub": {
"anthropic--claude-3-sonnet": {
"destinationName": "GenAIHubDestination",
"deploymentUrl": "/v2/inference/deployments/d3b...6575",
"resourceGroup": "default",
"anthropic_version": "bedrock-2023-05-31",
"max_tokens": 4096,
"modelName": "anthropic--claude-3-sonnet"
}
},
"GenAIHubDestination": {
"kind": "rest",
"credentials": {
"destination": "ai_core_dest_4hybridmode",
"requestTimeout": "300000"
}
}
}
}
Next, I configured a Lite destination instance and configured a new destination titled `ai_core_dest_4hybridmode` . Following tutorial here and used credentials from my AI core instance.
Since I am developing locally in Hybrid mode, I am binding the local CAP deployment to my destination in the cloud using which creates an entry in `.cdsrc-private.json` in the root folder of my project.
cf create-service destination lite cpapp-destination
cf create-service-key cpapp-destination cpapp-destination-key
cds bind -2 cpapp-destination:cpapp-destination-key
Next, based on Akash's input - used the following data for BTP destination:
Name: ai_core_dest_4hybridmode Description: SAP AI Core deployed service (generative AI hub) URL: <AI-API-INSIDE-AI-CORE-SERVICE-KEY> Type: HTTP ProxyType: Internet Authentication: OAuth2ClientCredentials tokenServiceURL: <TOKEN-SERVICE-URL-OF-AI-CORE-SERVICE-KEY>/oauth/token clientId: <YOUR-CLIENT-ID-OF-AI-CORE-SERVICE-KEY> clientSecret: <YOUR-CLIENT-SECRET-OF-AI-CORE-SERVICE-KEY> # Additional Properties: URL.headers.AI-Resource-Group: default # adjust if necessary URL.headers.Content-Type: application/json HTML5.DynamicDestination: true
Note the trailing `/oauth/token` at the end of the tokenServiceURL.
Use
cds watch --profile hybrid
For reference my service.js is:
const cds = require('@sap/cds')
const logger = cds.log('llm-invoke')
module.exports = cds.service.impl(
function () {
logger.log('-------------')
logger.log('-------this is an anonymous function invoking llm via GenAI hub------')
logger.log('-------------')
this.on('invoke', async (req) => {
let messages = [{
"role": "user",
"content": "Hello, Claude"
}]
// Reference: https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/consume-generative-ai-models-using-sap-ai-core#aws-bedrock
// Reference: https://github.com/ashivadi/btp-gen-ai-hub-sdk-samples/blob/main/docs/notebook-samples/5-invoke-bedrock-models-via-genaihub.ipynb
const chatCompletionReqPayload = {
"anthropic_version": "bedrock-2023-05-31",
"max_tokens": 512,
"messages": messages,
};
const chatModelConfig = cds.env.requires["gen-ai-hub"]["anthropic--claude-3.5-sonnet"];
/* just for reference, chatModelConfig from package.json is:
{
"destinationName": "GenAIHubDestination",
"deploymentUrl": "/v2/inference/deployments/d3bx.....58475",
"resourceGroup": "default",
"anthropic_version": "bedrock-2023-05-31",
"max_tokens": 4096,
"modelName": "anthropic--claude-3-sonnet"
}
*/
const llmBedrockPluginViaCAPLLM = await cds.connect.to("cap-llm-plugin");
let result = await llmBedrockPluginViaCAPLLM.getChatCompletionWithConfig(chatModelConfig, chatCompletionReqPayload);
logger.log('-------Result------')
logger.log(result)
logger.log('-------------')
return result;
});
}
)
Note,
It is expected that you get a 404:Not Found at the check Connection in my destination? My configuration works even with the check saying 404.
| User | Count |
|---|---|
| 19 | |
| 16 | |
| 3 | |
| 2 | |
| 2 | |
| 2 | |
| 1 | |
| 1 | |
| 1 | |
| 1 |
You must be a registered user to add a comment. If you've already registered, sign in. Otherwise, register and sign in.