Calling openssl from x64 application

[rlebeau]

Anything else we can check?

Do you have the same error if you use EVP_PKEY_new() to allocate the EVP_PKEY dynamically instead of declaring it statically on the stack? Eventually, Indy will be updated to support OpenSSL 1.1.x and you will have to do this anyway, since OpenSSL 1.1.0+ now uses opaque structures that must be allocated dynamically, so you may as well prepare for this sooner rather than later.

Try something more like this:

Code:

function TOAuth1SignatureMethod_RSA_SHA1.Hash_HMAC_SHA1(const AData, AKey: string): string;
const
  cPrivateKeyBegin = '-----BEGIN PRIVATE KEY-----';
  cPrivateKeyEnd = '-----END PRIVATE KEY-----';
var
  buffer: TBytes;
  clean_key: string;
  mdLength: TIdC_UInt;
  mdctx: PEVP_MD_CTX;
  KeyBuffer: PBIO;
  FRSA: PRSA;
  md: PEVP_MD;
  key: PEVP_PKEY;
  rc: Integer;
  iBegin, iEnd: Integer;
begin
  if not IdSSLOpenSSL.LoadOpenSSLLibrary then
    raise Exception.Create('LoadOpenSSLLibrary failed');

  // Load private key (key must include header and footer)
  //-----BEGIN PRIVATE KEY-----
  // ....
  // ....
  //-----END PRIVATE KEY-----

  iBegin := AKey.IndexOf(cPrivateKeyBegin);
  if iBegin < 0 then
    raise Exception.Create('Private key error');

  iEnd := AKey.IndexOf(cPrivateKeyEnd, iBegin + Length(cPrivateKeyBegin));
  if iEnd < 0 then
    raise Exception.Create('Private key error');
  Inc(iEnd, Length(cPrivateKeyEnd));

  clean_key := AKey.Substring(iBegin, iEnd - iBegin);
  buffer := TEncoding.ANSI.GetBytes(clean_key);

  md := EVP_get_digestbyname('SHA1');
  if md = nil then
    raise Exception.Create('SHA1 lookup error');

  mdctx := EVP_MD_CTX_create();
  if mdctx = nil then
    raise Exception.Create('MD_CTX out of memory');

  try
    rc := EVP_DigestInit(mdctx, md);
    if rc <> 1 then
      raise Exception.Create('EVP_DigestInit failed: ' + rc.ToString);

    KeyBuffer := BIO_new_mem_buf(PByte(buffer), Length(buffer));
    if KeyBuffer = nil then
      raise Exception.Create('RSA out of memory');

    try
      FRSA := PEM_read_bio_RSAPrivateKey(KeyBuffer, nil, nil, nil);
      if FRSA = nil then
        raise Exception.Create('Private key error');
    finally
      BIO_free(KeyBuffer);
    end;

    try
      key := EVP_PKEY_new();
      if key = nil then
        raise Exception.Create('PKEY out of memory');

      try
        rc := EVP_PKEY_set1_RSA(key, FRSA);
        if rc <> 1 then
          raise Exception.Create('EVP_PKEY_set1_RSA failed: ' + rc.ToString);

        rc := EVP_DigestSignInit(mdctx, nil, md, nil, key);
        if rc <> 1 then
          raise Exception.Create('EVP_DigestSignInit failed: ' + rc.ToString);

        buffer := TEncoding.ANSI.GetBytes(AData);

        rc := EVP_DigestSignUpdate(mdctx, PByte(buffer), Length(buffer));
        if rc <> 1 then
          raise Exception.Create('EVP_DigestSignUpdate failed: ' + rc.ToString);

        rc := EVP_DigestSignFinal(mdctx, nil, @mdLength);
        if rc <> 1 then
          raise Exception.Create('EVP_DigestFinal failed: ' + rc.ToString);

        SetLength(buffer, mdLength);

        rc := EVP_DigestSignFinal(mdctx, PIdAnsiChar(PByte(buffer)), @mdLength);
        if rc <> 1 then
          raise Exception.Create('EVP_DigestFinal failed: ' + rc.ToString);
      finally
        EVP_PKEY_free(key);
      end;
    finally
      RSA_free(FRSA);
    end;
  finally
    EVP_MD_CTX_destroy(mdctx);
  end;

  Result := TNetEncoding.Base64.EncodeBytesToString(PByte(buffer), mdLength);
  Result := Result.Replace(#13#10, '', [rfReplaceAll]);
end;

BTW, why are you implementing an HMAC SHA1 hash manually? Indy does have a TIdHMACSHA1 class available for that (as well as a TIdEncoderMIME class for encoding data to base64, without line breaks). OpenSSL also has its own HMAC functions, too.