verified，it's OK

the PR has beed verified,but there are still some problems, which are explained here:
1.many header files (*math.h)in the Xizi and CMSIS are currently coupled, so cmSIS is not supported in Xizi
2.the lcd driver is not supported in xizi and nuttx
3.the OV2640 driver is not supported in xizi and nuttx.

APP_Framework/Applications/knowing_app/cmsis_5_demo/Kconfig

@@ -11,6 +11,11 @@ menuconfig USING_CMSIS_5_DEMOAPP
             select IMAGE_PROCESSING_USING_TJPGD
             default n
 
+        config USING_CMSIS_5_NN_DEMOAPP_VEG_CLASSIFY
+            bool "Using CMSIS-5 NN demo app vegetable classify"
+            select USING_IMAGE_PROCESSING
+            select IMAGE_PROCESSING_USING_TJPGD
+            default n
     endif
 
 

APP_Framework/Applications/knowing_app/cmsis_5_demo/cmsisnn_vegetable_classify/README.md

@@ -0,0 +1,37 @@
+# CMSIS-NN vegetable classify example
+
+This example uses CMSIS-NN to classify vegetable in real time under certain circumstances .
+
+## Requirements:
+- CMSIS-NN in Framework/knowing/cmsis_5
+- **ov2640 need to be configured** in menuconfig "More Drivers->ov2640 driver" as follows
+  - Output format (RGB565 mode)
+  - (256) X direction resolution of outputimage
+  - (256) Y direction resolution of outputimage
+  - (512) X direction WINDOWS SIZE
+  - (512) Y direction WINDOWS SIZE
+
+## To run this demo:
+- Set up and configure the corresponding hardware environment.
+
+- Run demo by type the command
+  ``` 
+  cmsisnn_vegetable_classify
+
+## Results
+
+- **tomato**
+
+![tomato](https://www.gitlink.org.cn/repo/WentaoWong/xiuos/raw/branch/dev/APP_Framework/Applications/knowing_app/cmsis_5_demo/cmsisnn_vegetable_classify/doc/tomato.jpg)
+
+- **potato**
+
+![potato](https://www.gitlink.org.cn/repo/WentaoWong/xiuos/raw/branch/dev/APP_Framework/Applications/knowing_app/cmsis_5_demo/cmsisnn_vegetable_classify/doc/potato.jpg)
+
+- **pepper**
+
+![pepper](https://www.gitlink.org.cn/repo/WentaoWong/xiuos/raw/branch/dev/APP_Framework/Applications/knowing_app/cmsis_5_demo/cmsisnn_vegetable_classify/doc/pepper.jpg)
+
+- **mushroom**
+
+![mushroom](https://www.gitlink.org.cn/repo/WentaoWong/xiuos/raw/branch/dev/APP_Framework/Applications/knowing_app/cmsis_5_demo/cmsisnn_vegetable_classify/doc/mushroom.jpg)

APP_Framework/Applications/knowing_app/cmsis_5_demo/cmsisnn_vegetable_classify/SConscript

@@ -0,0 +1,18 @@
+from building import *
+import os
+
+cwd = GetCurrentDir()
+
+src = Split('''
+model/nn_vegetable_classify.c
+cmsisnn_vegetable_classify.c
+''')
+
+path = [
+    cwd + '/model',
+    cwd + '/demo'
+    ]
+
+group = DefineGroup('CMSISNN vegetable classify application', src, depend = ['USING_CMSIS_5_NN_DEMOAPP_VEG_CLASSIFY'], CPPPATH = path)
+
+Return('group')

APP_Framework/Applications/knowing_app/cmsis_5_demo/cmsisnn_vegetable_classify/cmsisnn_vegetable_classify.c

@@ -0,0 +1,188 @@
+#include <rtthread.h>
+#include <rtdevice.h>
+#include "stdio.h"
+#include "string.h"
+
+#ifdef OV2640_RGB565_MODE
+#ifdef RT_USING_POSIX
+#include <dfs_posix.h>
+#include <dfs_poll.h>
+#ifdef RT_USING_POSIX_TERMIOS
+#include <posix_termios.h>
+#endif
+#endif
+
+#include <drv_ov2640.h>
+#include <drv_lcd.h>
+#include "nn_vegetable_classify.h"
+#define JPEG_BUF_SIZE (2 * OV2640_X_RESOLUTION_IMAGE_OUTSIZE * OV2640_Y_RESOLUTION_IMAGE_OUTSIZE)
+#define IOCTL_ERROR 1
+
+static int fd = 0;
+static int infer_times = 0;
+static int photo_times = 0;
+static int height = OV2640_X_RESOLUTION_IMAGE_OUTSIZE;
+static int width = OV2640_Y_RESOLUTION_IMAGE_OUTSIZE;
+static _ioctl_shoot_para shoot_para_t = {0};
+const char *vegetable_label[] = {"mushroom", "pepper", "potato", "tomato"};
+
+uint8_t *resized_buffer = NULL;
+uint8_t *in_buffer = NULL;
+
+int get_top_prediction_detection(q7_t *predictions)
+{
+  int max_ind = 0;
+  int max_val = -128;
+  for (int i = 0; i < 10; i++)
+  {
+    if (max_val < predictions[i])
+    {
+      max_val = predictions[i];
+      max_ind = i;
+    }
+  }
+  return max_ind;
+}
+
+int cmsisnn_inference_vegetable_classify(uint8_t *input_data)
+{
+  int8_t output_data[4];
+  char output[50] = {0};
+  char outputPrediction[50] = {0};
+  memset(output, 0, 50);
+  memset(outputPrediction, 0, 50);
+
+  run_nn_sn_classify((int8_t *)input_data, output_data);
+  arm_softmax_q7(output_data, 4, output_data);
+
+  infer_times++;
+  int top_ind = get_top_prediction_detection(output_data);
+  printf("times:%d  Prediction:%s \r\n", infer_times, vegetable_label[top_ind]);
+  sprintf(outputPrediction, "times:%d Prediction:%s        \r\n", infer_times, vegetable_label[top_ind]);
+  lcd_show_string(1, 280, 240, 16, 16, outputPrediction, RED);
+  return top_ind;
+}
+
+void resize_rgb888in_rgb565out(uint8_t *camera_image, uint16_t *resize_image)
+{
+  uint8_t *psrc_temp = (uint8_t *)camera_image;
+  uint16_t *pdst_temp = (uint16_t *)resize_image;
+  for (int y = 0; y < height; y++)
+  {
+    for (int x = 0; x < width; x++)
+    {
+      *pdst_temp++ = (*psrc_temp++ & 0xF8) << 8 | (*psrc_temp++ & 0xFC) << 3 | *psrc_temp++ >> 3;
+    }
+  }
+}
+
+void resize_rgb565in_rgb888out(uint8_t *camera_image, uint8_t *resize_image)
+{
+  uint8_t *psrc_temp = (uint8_t *)camera_image;
+  uint8_t *pdst_temp = (uint8_t *)resize_image;
+  for (int y = 0; y < height; y++)
+  {
+    for (int x = 0; x < width; x++)
+    {
+      uint8_t pixel_lo = *psrc_temp++;
+      uint8_t pixel_hi = *psrc_temp++;
+      *pdst_temp++ = (0xF8 & pixel_hi);
+      *pdst_temp++ = ((0x07 & pixel_hi) << 5) | ((0xE0 & pixel_lo) >> 3);
+      *pdst_temp++ = (0x1F & pixel_lo) << 3;
+    }
+  }
+}
+
+void lcd_show_ov2640_thread_detection(uint8_t *rgbbuffer)
+{
+  int32_t ret = 0;
+  while (1)
+  {
+    ret = ioctl(fd, IOCTRL_CAMERA_START_SHOT, &shoot_para_t);
+    if (ret == IOCTL_ERROR)
+    {
+      printf("ov2640 can't wait event flag");
+      free(rgbbuffer);
+      return;
+    }
+    lcd_fill_array(0, 0, OV2640_X_RESOLUTION_IMAGE_OUTSIZE, OV2640_Y_RESOLUTION_IMAGE_OUTSIZE, rgbbuffer);
+
+    if (photo_times % 20 == 0)
+    {
+      resize_rgb565in_rgb888out(rgbbuffer, resized_buffer);
+      int pixel = 0;
+      for (int i = 0; i < 3 * width; i += 3 * width / CONV1_IN_DIM)
+      {
+        for (int j = 0; j < 3 * height; j += 3 * height / CONV1_IN_DIM)
+        {
+          for (int k = 0; k < 3; k++, pixel++)
+          {
+            *(in_buffer + pixel) = *(resized_buffer + 256 * i + j + k);
+          }
+        }
+      }
+      cmsisnn_inference_vegetable_classify(in_buffer);
+    }
+    photo_times++;
+  }
+}
+
+void cmsisnn_vegetable_classify()
+{
+  fd = open("/dev/ov2640", O_RDONLY);
+  if (fd < 0)
+  {
+    printf("open ov2640 fail !!");
+    return;
+  }
+  printf("memory_init \n\r");
+  uint8_t *JpegBuffer = malloc(JPEG_BUF_SIZE);
+  if (JpegBuffer == NULL)
+  {
+    printf("JpegBuffer senddata buf malloc error!\n");
+    return;
+  }
+  resized_buffer = malloc(3 * width * height);
+  if (resized_buffer == NULL)
+  {
+    printf("Resized_buffer buf malloc error!\n");
+    return;
+  }
+  in_buffer = malloc(CONV1_IN_CH * CONV1_IN_DIM * CONV1_IN_DIM);
+  if (in_buffer == NULL)
+  {
+    printf("In_buffer buf malloc error!\n");
+    return;
+  }
+  memory_init();
+  printf("memory_init success\n\r");
+  
+  shoot_para_t.pdata = (uint32_t)JpegBuffer;
+  shoot_para_t.length = JPEG_BUF_SIZE / 2;
+
+  int result = 0;
+  pthread_t tid = 0;
+  pthread_attr_t attr;
+  struct sched_param prio;
+  prio.sched_priority = 8;
+  size_t stack_size = 1024 * 11;
+  pthread_attr_init(&attr);
+  pthread_attr_setschedparam(&attr, &prio); 
+  pthread_attr_setstacksize(&attr, stack_size);
+
+  result = pthread_create(&tid, &attr, lcd_show_ov2640_thread_detection, JpegBuffer);
+  if (0 == result) {
+      printf("thread_detect_entry successfully!\n");
+  } else {
+      printf("thread_detect_entry failed! error code is %d.\n", result);
+      close(fd);
+  }
+
+  return;
+}
+
+#ifdef __RT_THREAD_H__
+MSH_CMD_EXPORT(cmsisnn_vegetable_classify, classify vegetable using cmsis-nn);
+#endif
+
+#endif

APP_Framework/Applications/knowing_app/cmsis_5_demo/cmsisnn_vegetable_classify/model/nn_vegetable_classify.c

@@ -0,0 +1,108 @@
+#include "nn_vegetable_classify.h"
+
+static const q7_t conv1_w[CONV1_WT_SHAPE] = CONV1_WT;
+static const q7_t conv1_b[CONV1_BIAS_SHAPE] = CONV1_BIAS;
+static const q7_t conv2_w[CONV2_WT_SHAPE] = CONV2_WT;
+static const q7_t conv2_b[CONV2_BIAS_SHAPE] = CONV2_BIAS;
+// static const q7_t conv3_w[CONV3_WT_SHAPE] =  CONV3_WT;
+// static const q7_t conv3_b[CONV3_BIAS_SHAPE] = CONV3_BIAS;
+static const q7_t interface_w[INTERFACE_WT_SHAPE] = INTERFACE_WT;
+static const q7_t interface_b[INTERFACE_BIAS_SHAPE] = INTERFACE_BIAS;
+static const q7_t linear_w[LINEAR_WT_SHAPE] = LINEAR_WT;
+static const q7_t linear_b[LINEAR_BIAS_SHAPE] = LINEAR_BIAS;
+
+q7_t *conv1_out = NULL;
+q7_t *pool1_out = NULL;
+q7_t *conv2_out = NULL;
+q7_t *pool2_out = NULL;
+//  q7_t *conv3_out = NULL ;
+q7_t *interface_out = NULL;
+q7_t *linear_out = NULL;
+q7_t *y_out = NULL;
+q7_t *conv_buffer = NULL;
+q7_t *fc_buffer = NULL;
+
+void memory_init()
+{
+  static int flag = 0;
+  if (flag == 0)
+  {
+    conv1_out = malloc(CONV1_OUT_CH * CONV1_OUT_DIM * CONV1_OUT_DIM);
+    if (conv1_out == NULL)
+    {
+      printf("conv1_out malloc failed...\n");
+      return;
+    }
+    pool1_out = malloc(CONV1_OUT_CH * POOL1_OUT_DIM * POOL1_OUT_DIM);
+    if (pool1_out == NULL)
+    {
+      printf("pool1_out malloc failed...\n");
+      return;
+    }
+    conv2_out = malloc(CONV2_OUT_CH * CONV2_OUT_DIM * CONV2_OUT_DIM);
+    if (conv2_out == NULL)
+    {
+      printf("conv2_out malloc failed...\n");
+      return;
+    }
+    pool2_out = malloc(CONV2_OUT_CH * POOL2_OUT_DIM * POOL2_OUT_DIM);
+    if (pool2_out == NULL)
+    {
+      printf("pool2_out malloc failed...\n");
+      return;
+    }
+    interface_out = malloc(INTERFACE_OUT_DIM);
+    if (interface_out == NULL)
+    {
+      printf("interface_out malloc failed...\n");
+      return;
+    }
+    linear_out = malloc(LINEAR_OUT_DIM);
+    if (linear_out == NULL)
+    {
+      printf("linear_out malloc failed...\n");
+      return;
+    }
+    y_out = malloc(LINEAR_OUT_DIM);
+    if (y_out == NULL)
+    {
+      printf("y_out malloc failed...\n");
+      return;
+    }
+    conv_buffer = malloc(MAX_CONV_BUFFER_SIZE);
+    if (conv_buffer == NULL)
+    {
+      printf("conv_buffer malloc failed...\n");
+      return;
+    }
+    fc_buffer = malloc(MAX_FC_BUFFER);
+    if (fc_buffer == NULL)
+    {
+      printf("fc_buffer malloc failed...\n");
+      return;
+    }
+  }
+}
+
+void run_nn_sn_classify(q7_t *input_data, q7_t *output_data)
+{
+  for (int i = 0; i < CONV1_IN_CH * CONV1_IN_DIM * CONV1_IN_DIM; i++)
+  {
+    input_data[i] = input_data[i] - 127;
+  }
+  arm_convolve_HWC_q7_basic(input_data, CONV1_IN_DIM, CONV1_IN_CH, conv1_w, CONV1_OUT_CH, CONV1_KER_DIM, CONV1_PAD, CONV1_STRIDE, conv1_b, CONV1_BIAS_LSHIFT, CONV1_OUT_RSHIFT, conv1_out, CONV1_OUT_DIM, (q15_t *)conv_buffer, fc_buffer);
+  arm_maxpool_q7_HWC(conv1_out, POOL1_IN_DIM, POOL1_IN_CH, POOL1_KER_DIM, POOL1_PAD, POOL1_STRIDE, POOL1_OUT_DIM, NULL, pool1_out);
+  arm_relu_q7(pool1_out, POOL1_OUT_DIM * POOL1_OUT_DIM * CONV1_OUT_CH);
+  arm_convolve_HWC_q7_basic(pool1_out, CONV2_IN_DIM, CONV2_IN_CH, conv2_w, CONV2_OUT_CH, CONV2_KER_DIM, CONV2_PAD, CONV2_STRIDE, conv2_b, CONV2_BIAS_LSHIFT, CONV2_OUT_RSHIFT, conv2_out, CONV2_OUT_DIM, (q15_t *)conv_buffer, NULL);
+  arm_maxpool_q7_HWC(conv2_out, POOL2_IN_DIM, POOL2_IN_CH, POOL2_KER_DIM, POOL2_PAD, POOL2_STRIDE, POOL2_OUT_DIM, NULL, pool2_out);
+  arm_relu_q7(pool2_out, POOL2_OUT_DIM * POOL2_OUT_DIM * CONV2_OUT_CH);
+  // printf("1\n");
+  // arm_convolve_HWC_q7_basic(pool2_out, CONV3_IN_DIM, CONV3_IN_CH, conv3_w, CONV3_OUT_CH, CONV3_KER_DIM,
+	// 					  CONV3_PAD, CONV3_STRIDE, conv3_b, CONV3_BIAS_LSHIFT, CONV3_OUT_RSHIFT, conv3_out,
+	// 					  CONV3_OUT_DIM, (q15_t *) conv_buffer, NULL);
+  // arm_relu_q7(conv3_out, CONV3_OUT_DIM * CONV3_OUT_DIM * CONV3_OUT_CH);
+  // printf("2\n");
+  arm_fully_connected_q7_opt(pool2_out, interface_w, INTERFACE_IN_DIM, INTERFACE_OUT_DIM, INTERFACE_BIAS_LSHIFT, INTERFACE_OUT_RSHIFT, interface_b, interface_out, (q15_t *)fc_buffer);
+  arm_relu_q7(interface_out, INTERFACE_OUT_DIM);
+  arm_fully_connected_q7_opt(interface_out, linear_w, LINEAR_IN_DIM, LINEAR_OUT_DIM, LINEAR_BIAS_LSHIFT, LINEAR_OUT_RSHIFT, linear_b, output_data, (q15_t *)fc_buffer);
+}

APP_Framework/Applications/knowing_app/cmsis_5_demo/cmsisnn_vegetable_classify/model/nn_vegetable_classify.h

@@ -0,0 +1,13 @@
+#ifndef __NN_H__
+#define __NN_H__
+
+#include <transform.h>
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+#include "parameter_vegetable_classify.h"
+#include "weights_vegetable_classify.h"
+
+void run_nn_detection(q7_t* input_data, q7_t* output_data);
+void memory_init();
+
+#endif

APP_Framework/Applications/knowing_app/cmsis_5_demo/cmsisnn_vegetable_classify/model/parameter_vegetable_classify.h

@@ -0,0 +1,56 @@
+#define CONV1_IN_CH 3
+#define CONV1_OUT_CH 32
+#define CONV1_KER_DIM 3
+#define CONV1_PAD 0
+#define CONV1_STRIDE 1
+#define CONV1_IN_DIM 32
+#define CONV1_OUT_DIM 30
+#define MAX_CONV_BUFFER_SIZE 3096
+#define POOL1_IN_CH 32
+#define POOL1_KER_DIM 2
+#define POOL1_PAD 0
+#define POOL1_STRIDE 2
+#define POOL1_IN_DIM 30
+#define POOL1_OUT_DIM 15
+#define CONV2_IN_CH 32
+#define CONV2_OUT_CH 32
+#define CONV2_KER_DIM 3
+#define CONV2_PAD 0
+#define CONV2_STRIDE 1
+#define CONV2_IN_DIM 15
+#define CONV2_OUT_DIM 13
+#define POOL2_IN_CH 32
+#define POOL2_KER_DIM 2
+#define POOL2_PAD 0
+#define POOL2_STRIDE 2
+#define POOL2_IN_DIM 13
+#define POOL2_OUT_DIM 6
+#define INTERFACE_OUT_DIM 32
+#define INTERFACE_IN_DIM 1152
+#define MAX_FC_BUFFER 3096
+#define LINEAR_OUT_DIM 4
+#define LINEAR_IN_DIM 32
+#define CONV1_BIAS_LSHIFT 6
+#define CONV1_OUT_RSHIFT 9
+#define CONV1_WEIGHT_Q 8
+#define CONV1_BIAS_Q 8
+#define CONV1_INPUT_Q 6
+#define CONV1_OUT_Q 5
+#define CONV2_BIAS_LSHIFT 4
+#define CONV2_OUT_RSHIFT 10
+#define CONV2_WEIGHT_Q 9
+#define CONV2_BIAS_Q 10
+#define CONV2_INPUT_Q 5
+#define CONV2_OUT_Q 4
+#define INTERFACE_BIAS_LSHIFT 3
+#define INTERFACE_OUT_RSHIFT 11
+#define INTERFACE_WEIGHT_Q 9
+#define INTERFACE_BIAS_Q 10
+#define INTERFACE_INPUT_Q 4
+#define INTERFACE_OUT_Q 2
+#define LINEAR_BIAS_LSHIFT 0
+#define LINEAR_OUT_RSHIFT 7
+#define LINEAR_WEIGHT_Q 7
+#define LINEAR_BIAS_Q 9
+#define LINEAR_INPUT_Q 2
+#define LINEAR_OUT_Q 2

APP_Framework/Framework/knowing/tensorflow-lite/tensorflow-lite-for-mcu/Makefile

@@ -83,8 +83,7 @@ ifeq ($(CONFIG_USING_TENSORFLOWLITEMICRO),y)
 		-Isource/third_party/ruy
 	DEFINES += -DTF_LITE_USE_GLOBAL_CMATH_FUNCTIONS \
 		-DTF_LITE_USE_GLOBAL_MAX \
-		-DTF_LITE_USE_GLOBAL_MIN \
-		-DTF_LITE_STATIC_MEMORY
+		-DTF_LITE_USE_GLOBAL_MIN
 endif
 
 include $(KERNEL_ROOT)/compiler.mk

APP_Framework/Framework/knowing/tensorflow-lite/tensorflow-lite-for-mcu/source/Makefile

@@ -23,8 +23,8 @@ $(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(SRCS)))
 
 LIBRARY_OBJS := $(filter-out tensorflow/lite/micro/examples/%, $(OBJS))
 
-CXXFLAGS += -std=c++11 -fno-rtti -fno-exceptions -fno-threadsafe-statics -fno-unwind-tables -ffunction-sections -fdata-sections -fmessage-length=0 -DTF_LITE_STATIC_MEMORY -DTF_LITE_DISABLE_X86_NEON -O3 -Werror -Wsign-compare -Wdouble-promotion -Wshadow -Wunused-variable -Wmissing-field-initializers -Wunused-function -Wswitch -Wvla -Wall -Wextra -Wstrict-aliasing -Wno-unused-parameter  -DCMSIS_NN -DTF_LITE_USE_CTIME -I. -I./third_party/gemmlowp -I./third_party/flatbuffers/include -I./third_party/ruy
-CCFLAGS +=  -std=c11 -fno-unwind-tables -ffunction-sections -fdata-sections -fmessage-length=0 -DTF_LITE_STATIC_MEMORY -DTF_LITE_DISABLE_X86_NEON -O3 -Werror -Wsign-compare -Wdouble-promotion -Wshadow -Wunused-variable -Wmissing-field-initializers -Wunused-function -Wswitch -Wvla -Wall -Wextra -Wstrict-aliasing -Wno-unused-parameter  -DCMSIS_NN -DTF_LITE_USE_CTIME -I. -I./third_party/gemmlowp -I./third_party/flatbuffers/include -I./third_party/ruy
+CXXFLAGS += -std=c++11 -fno-rtti -fno-exceptions -fno-threadsafe-statics -fno-unwind-tables -ffunction-sections -fdata-sections -fmessage-length=0 -DTF_LITE_DISABLE_X86_NEON -O3 -Werror -Wsign-compare -Wdouble-promotion -Wshadow -Wunused-variable -Wmissing-field-initializers -Wunused-function -Wswitch -Wvla -Wall -Wextra -Wstrict-aliasing -Wno-unused-parameter  -DCMSIS_NN -DTF_LITE_USE_CTIME -I. -I./third_party/gemmlowp -I./third_party/flatbuffers/include -I./third_party/ruy
+CCFLAGS +=  -std=c11 -fno-unwind-tables -ffunction-sections -fdata-sections -fmessage-length=0 -DTF_LITE_DISABLE_X86_NEON -O3 -Werror -Wsign-compare -Wdouble-promotion -Wshadow -Wunused-variable -Wmissing-field-initializers -Wunused-function -Wswitch -Wvla -Wall -Wextra -Wstrict-aliasing -Wno-unused-parameter  -DCMSIS_NN -DTF_LITE_USE_CTIME -I. -I./third_party/gemmlowp -I./third_party/flatbuffers/include -I./third_party/ruy
 
 LDFLAGS +=  -lm