From: michael Date: Sun, 18 Feb 2018 03:01:04 +0000 (-1000) Subject: Integrated speech recognition X-Git-Url: https://wolfpit.net/gitweb/?p=hardware%2Fbopit%2F.git;a=commitdiff_plain;h=HEAD Integrated speech recognition --- diff --git a/BopIt/App.config b/BopIt/App.config index 731f6de..896f669 100644 --- a/BopIt/App.config +++ b/BopIt/App.config @@ -1,6 +1,30 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/BopIt/BopIt.csproj b/BopIt/BopIt.csproj index 76d5aa0..00005c1 100644 --- a/BopIt/BopIt.csproj +++ b/BopIt/BopIt.csproj @@ -55,13 +55,73 @@ ..\packages\Accord.Video.3.8.0\lib\net46\Accord.Video.dll + + ..\packages\Google.Api.CommonProtos.1.1.0\lib\net45\Google.Api.CommonProtos.dll + + + ..\packages\Google.Api.Gax.2.1.0\lib\net45\Google.Api.Gax.dll + + + ..\packages\Google.Api.Gax.Grpc.2.1.0\lib\net45\Google.Api.Gax.Grpc.dll + + + ..\packages\Google.Apis.1.29.1\lib\net45\Google.Apis.dll + + + ..\packages\Google.Apis.Auth.1.29.1\lib\net45\Google.Apis.Auth.dll + + + ..\packages\Google.Apis.Auth.1.29.1\lib\net45\Google.Apis.Auth.PlatformServices.dll + + + ..\packages\Google.Apis.Core.1.29.1\lib\net45\Google.Apis.Core.dll + + + ..\packages\Google.Apis.1.29.1\lib\net45\Google.Apis.PlatformServices.dll + + + ..\packages\Google.Cloud.Speech.V1.1.0.1\lib\net45\Google.Cloud.Speech.V1.dll + + + ..\packages\Google.LongRunning.1.0.0\lib\net45\Google.LongRunning.dll + + + ..\packages\Google.Protobuf.3.4.1\lib\net45\Google.Protobuf.dll + + + ..\packages\Grpc.Auth.1.6.1\lib\net45\Grpc.Auth.dll + + + ..\packages\Grpc.Core.1.6.1\lib\net45\Grpc.Core.dll + ..\packages\NAudio.1.8.4\lib\net35\NAudio.dll + + ..\packages\Newtonsoft.Json.10.0.2\lib\net45\Newtonsoft.Json.dll + + + ..\packages\System.Interactive.Async.3.1.1\lib\net46\System.Interactive.Async.dll + + + ..\packages\System.Net.Http.4.3.1\lib\net46\System.Net.Http.dll + + + ..\packages\System.Security.Cryptography.Algorithms.4.3.0\lib\net461\System.Security.Cryptography.Algorithms.dll + + + ..\packages\System.Security.Cryptography.Encoding.4.3.0\lib\net46\System.Security.Cryptography.Encoding.dll + + + ..\packages\System.Security.Cryptography.Primitives.4.3.0\lib\net46\System.Security.Cryptography.Primitives.dll + + + ..\packages\System.Security.Cryptography.X509Certificates.4.3.0\lib\net461\System.Security.Cryptography.X509Certificates.dll + @@ -69,7 +129,6 @@ - @@ -115,5 +174,7 @@ This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + \ No newline at end of file diff --git a/BopIt/MainForm.Designer.cs b/BopIt/MainForm.Designer.cs index 70e01f8..c330424 100644 --- a/BopIt/MainForm.Designer.cs +++ b/BopIt/MainForm.Designer.cs @@ -28,12 +28,13 @@ /// private void InitializeComponent() { - System.Windows.Forms.DataVisualization.Charting.ChartArea chartArea1 = new System.Windows.Forms.DataVisualization.Charting.ChartArea(); - System.Windows.Forms.DataVisualization.Charting.Legend legend1 = new System.Windows.Forms.DataVisualization.Charting.Legend(); - System.Windows.Forms.DataVisualization.Charting.Series series1 = new System.Windows.Forms.DataVisualization.Charting.Series(); + System.Windows.Forms.DataVisualization.Charting.ChartArea chartArea2 = new System.Windows.Forms.DataVisualization.Charting.ChartArea(); + System.Windows.Forms.DataVisualization.Charting.Legend legend2 = new System.Windows.Forms.DataVisualization.Charting.Legend(); + System.Windows.Forms.DataVisualization.Charting.Series series2 = new System.Windows.Forms.DataVisualization.Charting.Series(); this.progressBar = new System.Windows.Forms.ProgressBar(); this.waveform = new System.Windows.Forms.DataVisualization.Charting.Chart(); this.tableLayoutPanel1 = new System.Windows.Forms.TableLayoutPanel(); + this.textBox1 = new System.Windows.Forms.TextBox(); ((System.ComponentModel.ISupportInitialize)(this.waveform)).BeginInit(); this.tableLayoutPanel1.SuspendLayout(); this.SuspendLayout(); @@ -45,41 +46,41 @@ this.progressBar.MarqueeAnimationSpeed = 1; this.progressBar.Maximum = 50; this.progressBar.Name = "progressBar"; - this.progressBar.Size = new System.Drawing.Size(1297, 96); + this.progressBar.Size = new System.Drawing.Size(1297, 45); this.progressBar.Step = 1; this.progressBar.TabIndex = 0; // // waveform // - chartArea1.AxisX.Interval = 2D; - chartArea1.AxisX.LabelStyle.Enabled = false; - chartArea1.AxisX.LineColor = System.Drawing.Color.DarkGray; - chartArea1.AxisX.MajorGrid.Enabled = false; - chartArea1.AxisX.MajorGrid.LineColor = System.Drawing.Color.LightGray; - chartArea1.AxisX.MajorTickMark.Enabled = false; - chartArea1.AxisY.Interval = 5D; - chartArea1.AxisY.LabelStyle.Enabled = false; - chartArea1.AxisY.LineColor = System.Drawing.Color.DarkGray; - chartArea1.AxisY.MajorGrid.Enabled = false; - chartArea1.AxisY.MajorGrid.LineColor = System.Drawing.Color.LightGray; - chartArea1.AxisY.MajorTickMark.Enabled = false; - chartArea1.AxisY.Maximum = 700D; - chartArea1.AxisY.Minimum = -700D; - chartArea1.BorderColor = System.Drawing.Color.LightGray; - chartArea1.Name = "ChartArea1"; - this.waveform.ChartAreas.Add(chartArea1); + chartArea2.AxisX.Interval = 2D; + chartArea2.AxisX.LabelStyle.Enabled = false; + chartArea2.AxisX.LineColor = System.Drawing.Color.DarkGray; + chartArea2.AxisX.MajorGrid.Enabled = false; + chartArea2.AxisX.MajorGrid.LineColor = System.Drawing.Color.LightGray; + chartArea2.AxisX.MajorTickMark.Enabled = false; + chartArea2.AxisY.Interval = 5D; + chartArea2.AxisY.LabelStyle.Enabled = false; + chartArea2.AxisY.LineColor = System.Drawing.Color.DarkGray; + chartArea2.AxisY.MajorGrid.Enabled = false; + chartArea2.AxisY.MajorGrid.LineColor = System.Drawing.Color.LightGray; + chartArea2.AxisY.MajorTickMark.Enabled = false; + chartArea2.AxisY.Maximum = 700D; + chartArea2.AxisY.Minimum = -700D; + chartArea2.BorderColor = System.Drawing.Color.LightGray; + chartArea2.Name = "ChartArea1"; + this.waveform.ChartAreas.Add(chartArea2); this.waveform.Dock = System.Windows.Forms.DockStyle.Fill; - legend1.Enabled = false; - legend1.Name = "Legend1"; - this.waveform.Legends.Add(legend1); - this.waveform.Location = new System.Drawing.Point(3, 105); + legend2.Enabled = false; + legend2.Name = "Legend1"; + this.waveform.Legends.Add(legend2); + this.waveform.Location = new System.Drawing.Point(3, 54); this.waveform.Name = "waveform"; - series1.ChartArea = "ChartArea1"; - series1.ChartType = System.Windows.Forms.DataVisualization.Charting.SeriesChartType.Line; - series1.Legend = "Legend1"; - series1.Name = "Series1"; - this.waveform.Series.Add(series1); - this.waveform.Size = new System.Drawing.Size(1297, 406); + series2.ChartArea = "ChartArea1"; + series2.ChartType = System.Windows.Forms.DataVisualization.Charting.SeriesChartType.Line; + series2.Legend = "Legend1"; + series2.Name = "Series1"; + this.waveform.Series.Add(series2); + this.waveform.Size = new System.Drawing.Size(1297, 353); this.waveform.TabIndex = 1; this.waveform.Text = "chart1"; // @@ -89,15 +90,27 @@ this.tableLayoutPanel1.ColumnStyles.Add(new System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Percent, 100F)); this.tableLayoutPanel1.Controls.Add(this.progressBar, 0, 0); this.tableLayoutPanel1.Controls.Add(this.waveform, 0, 1); + this.tableLayoutPanel1.Controls.Add(this.textBox1, 0, 2); this.tableLayoutPanel1.Dock = System.Windows.Forms.DockStyle.Fill; this.tableLayoutPanel1.Location = new System.Drawing.Point(0, 0); this.tableLayoutPanel1.Name = "tableLayoutPanel1"; - this.tableLayoutPanel1.RowCount = 2; + this.tableLayoutPanel1.RowCount = 3; + this.tableLayoutPanel1.RowStyles.Add(new System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 10F)); + this.tableLayoutPanel1.RowStyles.Add(new System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 70F)); this.tableLayoutPanel1.RowStyles.Add(new System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 20F)); - this.tableLayoutPanel1.RowStyles.Add(new System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 80F)); this.tableLayoutPanel1.Size = new System.Drawing.Size(1303, 514); this.tableLayoutPanel1.TabIndex = 2; // + // textBox1 + // + this.textBox1.Dock = System.Windows.Forms.DockStyle.Fill; + this.textBox1.Location = new System.Drawing.Point(3, 413); + this.textBox1.Multiline = true; + this.textBox1.Name = "textBox1"; + this.textBox1.ReadOnly = true; + this.textBox1.Size = new System.Drawing.Size(1297, 98); + this.textBox1.TabIndex = 2; + // // MainForm // this.AutoScaleDimensions = new System.Drawing.SizeF(8F, 16F); @@ -110,6 +123,7 @@ this.Load += new System.EventHandler(this.MainForm_Load); ((System.ComponentModel.ISupportInitialize)(this.waveform)).EndInit(); this.tableLayoutPanel1.ResumeLayout(false); + this.tableLayoutPanel1.PerformLayout(); this.ResumeLayout(false); } @@ -119,6 +133,7 @@ private System.Windows.Forms.ProgressBar progressBar; private System.Windows.Forms.DataVisualization.Charting.Chart waveform; private System.Windows.Forms.TableLayoutPanel tableLayoutPanel1; + private System.Windows.Forms.TextBox textBox1; } } diff --git a/BopIt/MainForm.cs b/BopIt/MainForm.cs index 056d0ab..b6609d2 100644 --- a/BopIt/MainForm.cs +++ b/BopIt/MainForm.cs @@ -1,8 +1,10 @@ -using System; +using Google.Cloud.Speech.V1; +using System; using System.Collections.Generic; using System.ComponentModel; using System.Data; using System.Drawing; +using System.IO; using System.IO.Ports; using System.Linq; using System.Text; @@ -36,11 +38,26 @@ namespace BopIt short[] waveValues = new short[(int)(e.BytesRecorded * .55)]; for (int i = 0 ; i < e.BytesRecorded; i += 2) waveValues[(i / 2) + ((waveValues.Length - e.BytesRecorded / 2) / 2)] = (short)(BitConverter.ToInt16(e.Buffer, i) / 50); - waveform.Invoke((MethodInvoker)(() => waveform.Series[0].Points.DataBindY(waveValues))); - UpdateVolumeMeter(Math.Abs(waveValues[waveValues.Length / 2])); + try + { + waveform.Invoke((MethodInvoker)(() => waveform.Series[0].Points.DataBindY(waveValues))); + UpdateVolumeMeter(Math.Abs(waveValues[waveValues.Length / 2])); + } + catch (Exception ex) + { + Console.WriteLine(ex.Message); + } }; waveIn.StartRecording(); + + + new Thread((ThreadStart)delegate + { + while (true) + StreamingMicRecognizeAsync(10).Wait(); + }).Start(); + //Volume meter de-incrimenter #region progressReset.Interval = 50; @@ -62,10 +79,103 @@ namespace BopIt if(data < progressBar.Maximum && data > progressBar.Value) progressBar.Invoke((MethodInvoker)(() => progressBar.Value = data)); } - private void MainForm_FormClosing(object sender, FormClosingEventArgs e) { progressReset.Dispose(); } + + private string currentSentance = String.Empty; + private string lastTrigger = String.Empty; + async Task StreamingMicRecognizeAsync(int seconds) + { + if (NAudio.Wave.WaveIn.DeviceCount < 1) + { + Console.WriteLine("No Mic"); + return -1; + } + var speech = SpeechClient.Create(); + var streamingCall = speech.StreamingRecognize(); + + // Write the initial request with the config. + await streamingCall.WriteAsync( + new StreamingRecognizeRequest() + { + StreamingConfig = new StreamingRecognitionConfig() + { + Config = new RecognitionConfig() + { + Encoding = + RecognitionConfig.Types.AudioEncoding.Linear16, + SampleRateHertz = 16000, + LanguageCode = "en", + }, + InterimResults = true, + } + }); + Task printResponses = Task.Run(async () => + { + while (await streamingCall.ResponseStream.MoveNext( + default(CancellationToken))) + { + foreach (var result in streamingCall.ResponseStream + .Current.Results) + { + var sentanceResult = result.Alternatives.Last().Transcript; + var newWordList = sentanceResult.Split(' '); + var currentWordList = currentSentance.Split(' '); + + if (newWordList.Length > currentWordList.Length || + newWordList.First() != currentWordList.First()) + { + currentSentance = sentanceResult; + textBox1.Invoke((MethodInvoker)(() => textBox1.AppendText(newWordList.Last() + Environment.NewLine))); + if ( newWordList.Last() != lastTrigger) + { + textBox1.Invoke((MethodInvoker)(() => textBox1.AppendText("TRIGGER: " + newWordList.Last() + Environment.NewLine))); + lastTrigger = newWordList.Last(); + } + Console.WriteLine(newWordList.Last()); + } + } + } + }); + + object writeLock = new object(); + bool writeMore = true; + var waveIn = new NAudio.Wave.WaveInEvent(); + waveIn.DeviceNumber = 0; + waveIn.WaveFormat = new NAudio.Wave.WaveFormat(16000, 1); + waveIn.DataAvailable += + (object sender, NAudio.Wave.WaveInEventArgs args) => + { + lock (writeLock) + { + if (!writeMore) return; + streamingCall.WriteAsync( + new StreamingRecognizeRequest() + { + AudioContent = Google.Protobuf.ByteString + .CopyFrom(args.Buffer, 0, args.BytesRecorded) + }).Wait(); + } + }; + waveIn.StartRecording(); + Console.WriteLine("Reconnecting stream"); + await Task.Delay(TimeSpan.FromSeconds(seconds)); + // Stop recording and shut down. + waveIn.StopRecording(); + lock (writeLock) writeMore = false; + await streamingCall.WriteCompleteAsync(); + await printResponses; + return 0; + } + + private static bool IsTrigger(string word) + { + foreach (var trigger in File.ReadAllLines("beeMovie.txt")) + if (word == trigger) + return true; + return false; + } } } diff --git a/BopIt/packages.config b/BopIt/packages.config index 4e392d3..7895f66 100644 --- a/BopIt/packages.config +++ b/BopIt/packages.config @@ -6,5 +6,23 @@ + + + + + + + + + + + + + + + + + + \ No newline at end of file