APP_Framework/Applications/:add CMSIS-NN vegetable classify example

APP_Framework/Applications/knowing_app/cmsis_5_demo/cmsisnn_vegetable_classify/cmsisnn_vegetable_classify.c

@@ -0,0 +1,188 @@
+#include <rtthread.h>
+#include <rtdevice.h>
+#include "stdio.h"
+#include "string.h"
+
+#ifdef OV2640_RGB565_MODE
+#ifdef RT_USING_POSIX
+#include <dfs_posix.h>
+#include <dfs_poll.h>
+#ifdef RT_USING_POSIX_TERMIOS
+#include <posix_termios.h>
+#endif
+#endif
+
+#include <drv_ov2640.h>
+#include <drv_lcd.h>
+#include "nn_vegetable_classify.h"
+#define JPEG_BUF_SIZE (2 * OV2640_X_RESOLUTION_IMAGE_OUTSIZE * OV2640_Y_RESOLUTION_IMAGE_OUTSIZE)
+#define IOCTL_ERROR 1
+
+static int fd = 0;
+static int infer_times = 0;
+static int photo_times = 0;
+static int height = OV2640_X_RESOLUTION_IMAGE_OUTSIZE;
+static int width = OV2640_Y_RESOLUTION_IMAGE_OUTSIZE;
+static _ioctl_shoot_para shoot_para_t = {0};
+const char *vegetable_label[] = {"mushroom", "pepper", "potato", "tomato"};
+
+uint8_t *resized_buffer = NULL;
+uint8_t *in_buffer = NULL;
+
+int get_top_prediction_detection(q7_t *predictions)
+{
+  int max_ind = 0;
+  int max_val = -128;
+  for (int i = 0; i < 10; i++)
+  {
+    if (max_val < predictions[i])
+    {
+      max_val = predictions[i];
+      max_ind = i;
+    }
+  }
+  return max_ind;
+}
+
+int cmsisnn_inference_vegetable_classify(uint8_t *input_data)
+{
+  int8_t output_data[4];
+  char output[50] = {0};
+  char outputPrediction[50] = {0};
+  memset(output, 0, 50);
+  memset(outputPrediction, 0, 50);
+
+  run_nn_sn_classify((int8_t *)input_data, output_data);
+  arm_softmax_q7(output_data, 4, output_data);
+
+  infer_times++;
+  int top_ind = get_top_prediction_detection(output_data);
+  printf("times:%d  Prediction:%s \r\n", infer_times, vegetable_label[top_ind]);
+  sprintf(outputPrediction, "times:%d Prediction:%s        \r\n", infer_times, vegetable_label[top_ind]);
+  lcd_show_string(1, 280, 240, 16, 16, outputPrediction, RED);
+  return top_ind;
+}
+
+void resize_rgb888in_rgb565out(uint8_t *camera_image, uint16_t *resize_image)
+{
+  uint8_t *psrc_temp = (uint8_t *)camera_image;
+  uint16_t *pdst_temp = (uint16_t *)resize_image;
+  for (int y = 0; y < height; y++)
+  {
+    for (int x = 0; x < width; x++)
+    {
+      *pdst_temp++ = (*psrc_temp++ & 0xF8) << 8 | (*psrc_temp++ & 0xFC) << 3 | *psrc_temp++ >> 3;
+    }
+  }
+}
+
+void resize_rgb565in_rgb888out(uint8_t *camera_image, uint8_t *resize_image)
+{
+  uint8_t *psrc_temp = (uint8_t *)camera_image;
+  uint8_t *pdst_temp = (uint8_t *)resize_image;
+  for (int y = 0; y < height; y++)
+  {
+    for (int x = 0; x < width; x++)
+    {
+      uint8_t pixel_lo = *psrc_temp++;
+      uint8_t pixel_hi = *psrc_temp++;
+      *pdst_temp++ = (0xF8 & pixel_hi);
+      *pdst_temp++ = ((0x07 & pixel_hi) << 5) | ((0xE0 & pixel_lo) >> 3);
+      *pdst_temp++ = (0x1F & pixel_lo) << 3;
+    }
+  }
+}
+
+void lcd_show_ov2640_thread_detection(uint8_t *rgbbuffer)
+{
+  int32_t ret = 0;
+  while (1)
+  {
+    ret = ioctl(fd, IOCTRL_CAMERA_START_SHOT, &shoot_para_t);
+    if (ret == IOCTL_ERROR)
+    {
+      printf("ov2640 can't wait event flag");
+      free(rgbbuffer);
+      return;
+    }
+    lcd_fill_array(0, 0, OV2640_X_RESOLUTION_IMAGE_OUTSIZE, OV2640_Y_RESOLUTION_IMAGE_OUTSIZE, rgbbuffer);
+
+    if (photo_times % 20 == 0)
+    {
+      resize_rgb565in_rgb888out(rgbbuffer, resized_buffer);
+      int pixel = 0;
+      for (int i = 0; i < 3 * width; i += 3 * width / CONV1_IN_DIM)
+      {
+        for (int j = 0; j < 3 * height; j += 3 * height / CONV1_IN_DIM)
+        {
+          for (int k = 0; k < 3; k++, pixel++)
+          {
+            *(in_buffer + pixel) = *(resized_buffer + 256 * i + j + k);
+          }
+        }
+      }
+      cmsisnn_inference_vegetable_classify(in_buffer);
+    }
+    photo_times++;
+  }
+}
+
+void cmsisnn_vegetable_classify()
+{
+  fd = open("/dev/ov2640", O_RDONLY);
+  if (fd < 0)
+  {
+    printf("open ov2640 fail !!");
+    return;
+  }
+  printf("memory_init \n\r");
+  uint8_t *JpegBuffer = malloc(JPEG_BUF_SIZE);
+  if (JpegBuffer == NULL)
+  {
+    printf("JpegBuffer senddata buf malloc error!\n");
+    return;
+  }
+  resized_buffer = malloc(3 * width * height);
+  if (resized_buffer == NULL)
+  {
+    printf("Resized_buffer buf malloc error!\n");
+    return;
+  }
+  in_buffer = malloc(CONV1_IN_CH * CONV1_IN_DIM * CONV1_IN_DIM);
+  if (in_buffer == NULL)
+  {
+    printf("In_buffer buf malloc error!\n");
+    return;
+  }
+  memory_init();
+  printf("memory_init success\n\r");
+  
+  shoot_para_t.pdata = (uint32_t)JpegBuffer;
+  shoot_para_t.length = JPEG_BUF_SIZE / 2;
+
+  int result = 0;
+  pthread_t tid = 0;
+  pthread_attr_t attr;
+  struct sched_param prio;
+  prio.sched_priority = 8;
+  size_t stack_size = 1024 * 11;
+  pthread_attr_init(&attr);
+  pthread_attr_setschedparam(&attr, &prio); 
+  pthread_attr_setstacksize(&attr, stack_size);
+
+  result = pthread_create(&tid, &attr, lcd_show_ov2640_thread_detection, JpegBuffer);
+  if (0 == result) {
+      printf("thread_detect_entry successfully!\n");
+  } else {
+      printf("thread_detect_entry failed! error code is %d.\n", result);
+      close(fd);
+  }
+
+  return;
+}
+
+#ifdef __RT_THREAD_H__
+MSH_CMD_EXPORT(cmsisnn_vegetable_classify, classify vegetable using cmsis-nn);
+#endif
+
+#endif

APP_Framework/Applications/knowing_app/cmsis_5_demo/cmsisnn_vegetable_classify/model/nn_vegetable_classify.c

@@ -0,0 +1,108 @@
+#include "nn_vegetable_classify.h"
+
+static const q7_t conv1_w[CONV1_WT_SHAPE] = CONV1_WT;
+static const q7_t conv1_b[CONV1_BIAS_SHAPE] = CONV1_BIAS;
+static const q7_t conv2_w[CONV2_WT_SHAPE] = CONV2_WT;
+static const q7_t conv2_b[CONV2_BIAS_SHAPE] = CONV2_BIAS;
+// static const q7_t conv3_w[CONV3_WT_SHAPE] =  CONV3_WT;
+// static const q7_t conv3_b[CONV3_BIAS_SHAPE] = CONV3_BIAS;
+static const q7_t interface_w[INTERFACE_WT_SHAPE] = INTERFACE_WT;
+static const q7_t interface_b[INTERFACE_BIAS_SHAPE] = INTERFACE_BIAS;
+static const q7_t linear_w[LINEAR_WT_SHAPE] = LINEAR_WT;
+static const q7_t linear_b[LINEAR_BIAS_SHAPE] = LINEAR_BIAS;
+
+q7_t *conv1_out = NULL;
+q7_t *pool1_out = NULL;
+q7_t *conv2_out = NULL;
+q7_t *pool2_out = NULL;
+//  q7_t *conv3_out = NULL ;
+q7_t *interface_out = NULL;
+q7_t *linear_out = NULL;
+q7_t *y_out = NULL;
+q7_t *conv_buffer = NULL;
+q7_t *fc_buffer = NULL;
+
+void memory_init()
+{
+  static int flag = 0;
+  if (flag == 0)
+  {
+    conv1_out = malloc(CONV1_OUT_CH * CONV1_OUT_DIM * CONV1_OUT_DIM);
+    if (conv1_out == NULL)
+    {
+      printf("conv1_out malloc failed...\n");
+      return;
+    }
+    pool1_out = malloc(CONV1_OUT_CH * POOL1_OUT_DIM * POOL1_OUT_DIM);
+    if (pool1_out == NULL)
+    {
+      printf("pool1_out malloc failed...\n");
+      return;
+    }
+    conv2_out = malloc(CONV2_OUT_CH * CONV2_OUT_DIM * CONV2_OUT_DIM);
+    if (conv2_out == NULL)
+    {
+      printf("conv2_out malloc failed...\n");
+      return;
+    }
+    pool2_out = malloc(CONV2_OUT_CH * POOL2_OUT_DIM * POOL2_OUT_DIM);
+    if (pool2_out == NULL)
+    {
+      printf("pool2_out malloc failed...\n");
+      return;
+    }
+    interface_out = malloc(INTERFACE_OUT_DIM);
+    if (interface_out == NULL)
+    {
+      printf("interface_out malloc failed...\n");
+      return;
+    }
+    linear_out = malloc(LINEAR_OUT_DIM);
+    if (linear_out == NULL)
+    {
+      printf("linear_out malloc failed...\n");
+      return;
+    }
+    y_out = malloc(LINEAR_OUT_DIM);
+    if (y_out == NULL)
+    {
+      printf("y_out malloc failed...\n");
+      return;
+    }
+    conv_buffer = malloc(MAX_CONV_BUFFER_SIZE);
+    if (conv_buffer == NULL)
+    {
+      printf("conv_buffer malloc failed...\n");
+      return;
+    }
+    fc_buffer = malloc(MAX_FC_BUFFER);
+    if (fc_buffer == NULL)
+    {
+      printf("fc_buffer malloc failed...\n");
+      return;
+    }
+  }
+}
+
+void run_nn_sn_classify(q7_t *input_data, q7_t *output_data)
+{
+  for (int i = 0; i < CONV1_IN_CH * CONV1_IN_DIM * CONV1_IN_DIM; i++)
+  {
+    input_data[i] = input_data[i] - 127;
+  }
+  arm_convolve_HWC_q7_basic(input_data, CONV1_IN_DIM, CONV1_IN_CH, conv1_w, CONV1_OUT_CH, CONV1_KER_DIM, CONV1_PAD, CONV1_STRIDE, conv1_b, CONV1_BIAS_LSHIFT, CONV1_OUT_RSHIFT, conv1_out, CONV1_OUT_DIM, (q15_t *)conv_buffer, fc_buffer);
+  arm_maxpool_q7_HWC(conv1_out, POOL1_IN_DIM, POOL1_IN_CH, POOL1_KER_DIM, POOL1_PAD, POOL1_STRIDE, POOL1_OUT_DIM, NULL, pool1_out);
+  arm_relu_q7(pool1_out, POOL1_OUT_DIM * POOL1_OUT_DIM * CONV1_OUT_CH);
+  arm_convolve_HWC_q7_basic(pool1_out, CONV2_IN_DIM, CONV2_IN_CH, conv2_w, CONV2_OUT_CH, CONV2_KER_DIM, CONV2_PAD, CONV2_STRIDE, conv2_b, CONV2_BIAS_LSHIFT, CONV2_OUT_RSHIFT, conv2_out, CONV2_OUT_DIM, (q15_t *)conv_buffer, NULL);
+  arm_maxpool_q7_HWC(conv2_out, POOL2_IN_DIM, POOL2_IN_CH, POOL2_KER_DIM, POOL2_PAD, POOL2_STRIDE, POOL2_OUT_DIM, NULL, pool2_out);
+  arm_relu_q7(pool2_out, POOL2_OUT_DIM * POOL2_OUT_DIM * CONV2_OUT_CH);
+  // printf("1\n");
+  // arm_convolve_HWC_q7_basic(pool2_out, CONV3_IN_DIM, CONV3_IN_CH, conv3_w, CONV3_OUT_CH, CONV3_KER_DIM,
+	// 					  CONV3_PAD, CONV3_STRIDE, conv3_b, CONV3_BIAS_LSHIFT, CONV3_OUT_RSHIFT, conv3_out,
+	// 					  CONV3_OUT_DIM, (q15_t *) conv_buffer, NULL);
+  // arm_relu_q7(conv3_out, CONV3_OUT_DIM * CONV3_OUT_DIM * CONV3_OUT_CH);
+  // printf("2\n");
+  arm_fully_connected_q7_opt(pool2_out, interface_w, INTERFACE_IN_DIM, INTERFACE_OUT_DIM, INTERFACE_BIAS_LSHIFT, INTERFACE_OUT_RSHIFT, interface_b, interface_out, (q15_t *)fc_buffer);
+  arm_relu_q7(interface_out, INTERFACE_OUT_DIM);
+  arm_fully_connected_q7_opt(interface_out, linear_w, LINEAR_IN_DIM, LINEAR_OUT_DIM, LINEAR_BIAS_LSHIFT, LINEAR_OUT_RSHIFT, linear_b, output_data, (q15_t *)fc_buffer);
+}

APP_Framework/Applications/knowing_app/cmsis_5_demo/cmsisnn_vegetable_classify/model/nn_vegetable_classify.h

@@ -0,0 +1,13 @@
+#ifndef __NN_H__
+#define __NN_H__
+
+#include <transform.h>
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+#include "parameter_vegetable_classify.h"
+#include "weights_vegetable_classify.h"
+
+void run_nn_detection(q7_t* input_data, q7_t* output_data);
+void memory_init();
+
+#endif

APP_Framework/Applications/knowing_app/cmsis_5_demo/cmsisnn_vegetable_classify/model/parameter_vegetable_classify.h

@@ -0,0 +1,56 @@
+#define CONV1_IN_CH 3
+#define CONV1_OUT_CH 32
+#define CONV1_KER_DIM 3
+#define CONV1_PAD 0
+#define CONV1_STRIDE 1
+#define CONV1_IN_DIM 32
+#define CONV1_OUT_DIM 30
+#define MAX_CONV_BUFFER_SIZE 3096
+#define POOL1_IN_CH 32
+#define POOL1_KER_DIM 2
+#define POOL1_PAD 0
+#define POOL1_STRIDE 2
+#define POOL1_IN_DIM 30
+#define POOL1_OUT_DIM 15
+#define CONV2_IN_CH 32
+#define CONV2_OUT_CH 32
+#define CONV2_KER_DIM 3
+#define CONV2_PAD 0
+#define CONV2_STRIDE 1
+#define CONV2_IN_DIM 15
+#define CONV2_OUT_DIM 13
+#define POOL2_IN_CH 32
+#define POOL2_KER_DIM 2
+#define POOL2_PAD 0
+#define POOL2_STRIDE 2
+#define POOL2_IN_DIM 13
+#define POOL2_OUT_DIM 6
+#define INTERFACE_OUT_DIM 32
+#define INTERFACE_IN_DIM 1152
+#define MAX_FC_BUFFER 3096
+#define LINEAR_OUT_DIM 4
+#define LINEAR_IN_DIM 32
+#define CONV1_BIAS_LSHIFT 6
+#define CONV1_OUT_RSHIFT 9
+#define CONV1_WEIGHT_Q 8
+#define CONV1_BIAS_Q 8
+#define CONV1_INPUT_Q 6
+#define CONV1_OUT_Q 5
+#define CONV2_BIAS_LSHIFT 4
+#define CONV2_OUT_RSHIFT 10
+#define CONV2_WEIGHT_Q 9
+#define CONV2_BIAS_Q 10
+#define CONV2_INPUT_Q 5
+#define CONV2_OUT_Q 4
+#define INTERFACE_BIAS_LSHIFT 3
+#define INTERFACE_OUT_RSHIFT 11
+#define INTERFACE_WEIGHT_Q 9
+#define INTERFACE_BIAS_Q 10
+#define INTERFACE_INPUT_Q 4
+#define INTERFACE_OUT_Q 2
+#define LINEAR_BIAS_LSHIFT 0
+#define LINEAR_OUT_RSHIFT 7
+#define LINEAR_WEIGHT_Q 7
+#define LINEAR_BIAS_Q 9
+#define LINEAR_INPUT_Q 2
+#define LINEAR_OUT_Q 2