PLUGIN: AGSSpeech - Speech Recognition and Synthesis

[Kweepa]

AGSSpeech

Speech Recognition and Synthesis plugin (requires .NET3.0+ and Vista or Win7)
Uses Microsoft Speech API 5.

Demo first I guess:
http://www.kweepa.com/step/ags/tech/QuiteAFlap_Speech.zip
The first time you run the game, it configures the SpeechRecognition engine. Once that is done, you have to click on the microphone to set it to Listening.
I modified my last HourGame, so it's not a great example, but it is completable!

And the download:
http://www.kweepa.com/step/ags/tech/AGSSpeech10.zip
Copy AGSSpeech.dll into the folder containing the editor.
Copy PluginApiWrapper.dll into the compiled and _Debug folders of your game.
Yes, brought to you by AGSPluginSharp!

Code: ags

function room_Load()
{
  // first string is the spoken string, second string is the semantic value (meaning)
  SpeechAddVerb("look at", "look at");
  SpeechAddVerb("examine", "look at");
  SpeechAddVerb("pick up", "use");
  SpeechAddVerb("use", "use");
  SpeechAddVerb("go to", "go to");
  SpeechAddNoun("toolbox", "toolbox");
  SpeechAddNoun("exit", "cliff");
  SpeechAddNoun("cliff", "cliff");
  SpeechAddNoun("machine", "cliff");
  SpeechRestart();
}

function Speak(String text)
{
  SpeechSpeakAsync(text);
  Display(text);
  SpeechSpeakAsyncCancel();
  // You can also use a blocking form:
  //SpeechSpeak(text);
}

function room_FirstLoad()
{
  Speak("It's taken you years, and your life savings, but you have finally constructed the machine your uncle Leo sketched.");
  Speak("You managed to contain your enthusiasm and wait till morning. Now to see if it flies.");
}


function room_RepExec()
{
  // this returns the spoken phrase
  String phrase = SpeechPeekPhrase();
  if (!String.IsNullOrEmpty(phrase))
  {
    // these return the semantic components
    // phrases can be of the form:
    // subject (noun)
    // verb
    // verb subject
    // verb subject on/with object (noun)
    String verb = SpeechGetVerb();
    String subject = SpeechGetSubject();
    String obj = SpeechGetObject();
    SpeechPopPhrase();

    // use the returned values
  }
}

[Calin Leafshade]

demo crashes D:

room 1: line 19

[Dualnames]

It worked for me. That's really impressive.

[Kweepa]

demo crashes D:

room 1: line 19

Rats!
(The crash is trying to start up Microsoft Speech.)
What is your OS and .NET version? What sort of microphone do you have?

Hooray for duals!

[Calin Leafshade]

Windows 7,
.NET 4.0
Bluetooth mic (Planetronics CS50)

EDIT: Also, speech recognition works fine on my computer and the game does seem to activate it before crashing.

[Kweepa]

Do you have .NET 3.5 too?
Could you try pasting this code into a new c# project (console application), then adding a reference to System.Speech and seeing what happens? Thanks Leafy!

Code: ags

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

using System.Speech.Recognition;

namespace SpeechRecognition
{
    class Program
    {
        static Dictionary<String, List<String>> mVerbs = new Dictionary<string, List<string>>();
        static Dictionary<String, List<String>> mNouns = new Dictionary<string, List<string>>();

        static SpeechRecognizer mRecognizer;

        private static void SpeechAddVerb(string verb, string semantic)
        {
            if (mVerbs.ContainsKey(semantic))
            {
                mVerbs[semantic].Add(verb);
            }
            else
            {
                List<string> val = new List<string>();
                val.Add(verb);
                mVerbs.Add(semantic, val);
            }
        }

        private static void SpeechAddNoun(string noun, string semantic)
        {
            if (mNouns.ContainsKey(semantic))
            {
                mNouns[semantic].Add(noun);
            }
            else
            {
                List<string> val = new List<string>();
                val.Add(noun);
                mNouns.Add(semantic, val);
            }
        }

        private static GrammarBuilder MakePartialGrammar(string semaKey, Dictionary<String, List<String>> stuff)
        {
            List<GrammarBuilder> smallList = new List<GrammarBuilder>();
            foreach (KeyValuePair<String, List<String>> pair in stuff)
            {
                foreach (string phrase in pair.Value)
                {
                    GrammarBuilder stuffBuilder = new GrammarBuilder();
                    stuffBuilder.Append(new SemanticResultValue(phrase, pair.Key));
                    smallList.Add(stuffBuilder);
                }
            }
            GrammarBuilder grammarBuilder = new GrammarBuilder();
            grammarBuilder.Append(new Choices(smallList.ToArray()));
            return new GrammarBuilder(new SemanticResultKey(semaKey, grammarBuilder));
        }

        static void Main(string[] args)
        {
            SpeechAddVerb("look at", "look at");
            SpeechAddVerb("examine", "look at");
            SpeechAddVerb("pick up", "use");
            SpeechAddVerb("use", "use");
            SpeechAddVerb("go to", "go to");
            SpeechAddNoun("tail", "tail");
            SpeechAddNoun("left wing", "left wing");
            SpeechAddNoun("right wing", "right wing");
            SpeechAddNoun("harness", "harness");
            SpeechAddNoun("cockpit", "harness");
            SpeechAddNoun("seat", "harness");
            SpeechAddNoun("workshop", "workshop");
            SpeechAddNoun("exit", "workshop");
            SpeechAddNoun("cliff", "cliff");

            GrammarBuilder verbGrammar = MakePartialGrammar("verb", mVerbs);
            GrammarBuilder blurbGrammar = new GrammarBuilder(new Choices("a", "an", "the"));
            GrammarBuilder subjectGrammar = MakePartialGrammar("subject", mNouns);
            GrammarBuilder withGrammar = new GrammarBuilder(new Choices("on", "with"));
            GrammarBuilder objectGrammar = MakePartialGrammar("object", mNouns);

            GrammarBuilder blurbSubjectGrammar = new GrammarBuilder();
            blurbSubjectGrammar.Append(blurbGrammar, 0, 1);
            blurbSubjectGrammar.Append(subjectGrammar);

            GrammarBuilder verbSubjectGrammar = new GrammarBuilder();
            verbSubjectGrammar.Append(verbGrammar);
            verbSubjectGrammar.Append(blurbSubjectGrammar);

            GrammarBuilder verbSubjectObjectGrammar = new GrammarBuilder();
            verbSubjectObjectGrammar.Append(verbSubjectGrammar);
            verbSubjectObjectGrammar.Append(withGrammar);
            verbSubjectObjectGrammar.Append(blurbGrammar, 0, 1);
            verbSubjectObjectGrammar.Append(objectGrammar);

            Choices phraseTypes = new Choices(new GrammarBuilder[] { verbGrammar, blurbSubjectGrammar, verbSubjectGrammar, verbSubjectObjectGrammar });
            GrammarBuilder grammarBuilder = new GrammarBuilder(phraseTypes);

            Grammar g = new Grammar(grammarBuilder);

            g.SpeechRecognized += new EventHandler<SpeechRecognizedEventArgs>(SpeechRecognized);

            mRecognizer = new SpeechRecognizer();
            mRecognizer.LoadGrammar(g);

            while (true) { }
        }

        private static void SpeechRecognized(object sender, RecognitionEventArgs args)
        {
            string mPhrase = "";
            string mVerb = "";
            string mSubject = "";
            string mObject = "";
            if (args.Result.Confidence > 0.3f)
            {
                mPhrase = args.Result.Text;
                SemanticValue sema = args.Result.Semantics;
                if (sema.ContainsKey("verb"))
                {
                    mVerb = (String)sema["verb"].Value;
                }
                if (sema.ContainsKey("subject"))
                {
                    mSubject = (String)sema["subject"].Value;
                }
                if (sema.ContainsKey("object"))
                {
                    mObject = (String)sema["object"].Value;
                }
            }
        }
    }
}

[Jubei23]

Hi, this is awesome.

I ran the demo fine, but I'm having trouble figuring out how to use the module in my own game. I've copied the DLLs as you specify in the forum post, but the functions that you've provided are unknown to the compiler when i try to use them in my global or room scripts.

Could someone please post a more detailed description of getting a spoken "hello world" in a new game.

Thanks!