FPGA实现红外图像的坏点去除的方法

1.背景

目前用到的国产红外探测器普遍均匀度较差、且存在较多坏点，为了不影响最终的成像质量一般都会对探测器输出的图像先进行均匀性矫正和坏点去除。

2.基本原理

坏点去除原理很简单就是用周围的像素值来代替坏点的像素值。首先需要判断一张图像中坏点的位置，用待标定红外相机拍摄不同温度的黑体图像，坏点对温度的响应明显区别于正常的像素点，将这些点坐标标记出来。坏点替换时一般根据实际情况用3*3或者5*5或者更大的开窗的像素平均值来替换中心坏点的值。在实际操作时用均值替换的效果不如用周围像素的中值替换，因为很多红外探测器坏点喜欢集中出现，一个坏点周围可能还有坏点，用中值替换可以减少周围坏点对替换后效果的影响。

3.FPGA实现

坏点替换算法和中值滤波或者均值滤波算法很相似，坏点算法仅对标记为坏点的开窗计算均值或中值进行替换，而中值滤波或是均值滤波则对整幅图都计算中值或均值进行替换。所以实现坏点算法的过程和滤波算法极为相似

首先建立3*3开窗的寄存器，将图像视频流用移位寄存器ip核缓存两行，这样三行图像就实现了并行输出了，和当前行输出一起缓存三次就形成了3*3开窗。注意这里缓存行的长度包含了行空闲，如果一行长度超出了移位寄存器（shift ram ip）的最大长度就用两个寄存器。FPGA的图像处理实现基本都建立在开窗基础之上。

将坏点标记和对应的图像像素一起缓存，中间的那行的中间像素若标记为1则代表当前开窗为坏点开窗，进行中值或均值替换，若不为1则不进行替换。

1.下面是以中值滤波为基础的坏点替换算法：

开窗建立顶层：

`timescale 1ns / 1ps
//
// Company: 
// Engineer: 
// 
// Create Date: 2022/07/19 14:08:30
// Design Name: 
// Module Name: mid_wave
// Project Name: 
// Target Devices: 
// Tool Versions: 
// Description: 
// 
// Dependencies: 
// 
// Revision:
// Revision 0.01 - File Created
// Additional Comments:
// 
//
 
 
module mid_wave(
   input I_clk,
   input I_reset,
   input I_clk_test,
   input I_frame_valid,
   input I_line_valid,
   input[31:0] I_par_Q,
   input[15:0] I_video_in,
   output wire O_frame_valid,
   output wire O_line_valid,
   output wire[15:0] O_video_out,
   output O_hblank,
   output O_vblank
    );
    
reg [18:0] S_temp0_r0;
reg [18:0] S_temp0_r1;
reg [18:0] S_temp0_r2;
reg [18:0] S_temp1_r0;
reg [18:0] S_temp1_r1;
reg [18:0] S_temp1_r2;
reg [18:0] S_temp2_r0;
reg [18:0] S_temp2_r1;
reg [18:0] S_temp2_r2;
wire[18:0] S_fifo0;
wire[18:0] S_fifo1;
wire[18:0] S_fifo0_1;
wire[18:0] S_fifo1_1;
wire[18:0] S_video_in;
wire[15:0] S_video_out;
wire S_blind_sig;
 
assign S_video_in = {I_frame_valid,I_line_valid,S_blind_sig,I_video_in};
assign O_hblank = (~O_line_valid) & O_frame_valid;
assign O_vblank = ~O_frame_valid;
assign S_blind_sig = I_par_Q[28];
 
 
c_shift_ram_0 c_shift_ram_0_0(
    .D(S_video_in),
    .CLK(I_clk),
    .Q(S_fifo0_1)
  );
  
c_shift_ram_0 c_shift_ram_0_1(
    .D(S_fifo0_1),
    .CLK(I_clk),
    .Q(S_fifo0)
  );
  
c_shift_ram_0 c_shift_ram_0_2(
    .D(S_fifo0),
    .CLK(I_clk),
    .Q(S_fifo1_1)
  );
  
c_shift_ram_0 c_shift_ram_0_3(
    .D(S_fifo1_1),
    .CLK(I_clk),
    .Q(S_fifo1)
  );
  
mid_data mid_data_i(
    //System Interfaces
    .sclk(I_clk)                 ,
    .rst_n(I_reset)           ,
    .mat_row1(S_video_in)     ,
    .mat_row2(S_fifo0)        ,
    .mat_row3(S_fifo1)        ,
    .O_frame(O_frame_valid)   ,
    .O_line(O_line_valid)     ,
    .O_data(O_video_out)         
);
 
///
 
 
endmodule

I_par_Q为坏点标记因为是matlab计算的浮点数所以是32位也可以只用1位来表示坏点。将坏点标记和行同步帧同步一起加到数据的高3位，再一起进行缓存便于最后输出。这里是640*512 16位红外图像，每一行加上行空闲一共有700多个像素时钟周期超出移位寄存器ip最大设置，所以每一行用了2个移位寄存器ip。

中值计算

`timescale 1ns / 1ps
//
// Company: 
// Engineer: 
// 
// Create Date: 2022/07/19 14:24:10
// Design Name: 
// Module Name: mid_data
// Project Name: 
// Target Devices: 
// Tool Versions: 
// Description: 
// 
// Dependencies: 
// 
// Revision:
// Revision 0.01 - File Created
// Additional Comments:
// 
//
 
 
module mid_data(
    //System Interfaces
    input                   sclk            ,
    input                   rst_n           ,
    //Communication Interfaces
    input           [18:0]  mat_row1        ,
    input           [18:0]  mat_row2        ,
    input           [18:0]  mat_row3        ,
    output          reg     O_frame         ,
    output            reg        O_line            ,
    output  reg     [15:0]  O_data         
);
 
//========================================================================================\
//**************Define Parameter and  Internal Signals**********************************
//========================================================================================/
 
reg                 [18:0]  mat_row1_1      ;
reg                 [18:0]  mat_row2_1      ;
reg                 [18:0]  mat_row3_1      ;
reg                 [18:0]  mat_row1_2      ;
reg                 [18:0]  mat_row2_2      ;
reg                 [18:0]  mat_row3_2      ;
                      
reg                 [18:0]  max_h1          ;
reg                 [18:0]  mid_h1          ;  
reg                 [18:0]  min_h1          ; 
reg                 [18:0]  max_h2          ;
reg                 [18:0]  mid_h2          ;  
reg                 [18:0]  min_h2          ;  
reg                 [18:0]  max_h3          ;
reg                 [18:0]  mid_h3          ;  
reg                 [18:0]  min_h3          ; 
reg                 [18:0]  min_max         ;
reg                 [18:0]  mid_mid         ;
reg                 [18:0]  max_min         ;       
  
 
 
//========================================================================================\
//**************     Main      Code        **********************************
//========================================================================================/
always @(posedge sclk)
    begin
        mat_row1_1          <=          mat_row1;
        mat_row2_1          <=          mat_row2;
        mat_row3_1          <=          mat_row3;
        mat_row1_2          <=          mat_row1_1;
        mat_row2_2          <=          mat_row2_1;
        mat_row3_2          <=          mat_row3_1;
    end
    
    
always @(posedge sclk or negedge rst_n)
begin
    if(rst_n == 1'b0)
        max_h1              <=          18'd0;        
    else if(mat_row1[15:0] >= mat_row1_1[15:0] && mat_row1[15:0] >= mat_row1_2[15:0])
        max_h1              <=          mat_row1;
    else if(mat_row1_1[15:0] >= mat_row1[15:0] && mat_row1_1[15:0] >= mat_row1_2[15:0]) 
        max_h1              <=          mat_row1_1;
    else
        max_h1              <=          mat_row1_2;
end
 
always @(posedge sclk or negedge rst_n)
begin
    if(rst_n == 1'b0)
        mid_h1              <=          18'd0;        
    else if((mat_row1[15:0] >= mat_row1_1[15:0] && mat_row1_1[15:0] >= mat_row1_2[15:0]) || (mat_row1_2[15:0] >= mat_row1_1[15:0] && mat_row1_1[15:0] >= mat_row1[15:0]))
        mid_h1              <=          mat_row1_1;
    else if((mat_row1_1[15:0] >= mat_row1[15:0] && mat_row1[15:0] >= mat_row1_2[15:0]) || (mat_row1_2[15:0] >= mat_row1[15:0] && mat_row1[15:0] >= mat_row1_1[15:0]))
        mid_h1              <=          mat_row1;
    else
        mid_h1              <=          mat_row1_2;
end
          
always @(posedge sclk or negedge rst_n)
begin
    if(rst_n == 1'b0)
        min_h1              <=          18'd0;        
    else if(mat_row1[15:0] <= mat_row1_1[15:0] && mat_row1[15:0] <= mat_row1_2[15:0])
        min_h1              <=          mat_row1;
    else if(mat_row1_1[15:0] <= mat_row1[15:0] && mat_row1_1[15:0] <= mat_row1_2[15:0]) 
        min_h1              <=          mat_row1_1;
    else
        min_h1              <=          mat_row1_2;  
end
 
always @(posedge sclk or negedge rst_n)
begin
    if(rst_n == 1'b0)
        max_h2              <=          18'd0;        
    else if(mat_row2[15:0] >= mat_row2_1[15:0] && mat_row2[15:0] >= mat_row2_2[15:0])
        max_h2              <=          mat_row2;
    else if(mat_row2_1[15:0] >= mat_row2[15:0] && mat_row2_1[15:0] >= mat_row2_2[15:0]) 
        max_h2              <=          mat_row2_1;
    else
        max_h2              <=          mat_row2_2;
end
 
always @(posedge sclk or negedge rst_n)
begin
    if(rst_n == 1'b0)
        mid_h2              <=          18'd0;        
    else if((mat_row2[15:0] >= mat_row2_1[15:0] && mat_row2_1[15:0] >= mat_row2_2[15:0]) || (mat_row2_2[15:0] >= mat_row2_1[15:0] && mat_row2_1[15:0] >= mat_row2[15:0]))
        mid_h2              <=          mat_row2_1;
    else if((mat_row2_1[15:0] >= mat_row2[15:0] && mat_row2[15:0] >= mat_row2_2[15:0]) || (mat_row2_2[15:0] >= mat_row2[15:0] && mat_row2[15:0] >= mat_row2_1[15:0]))
        mid_h2              <=          mat_row2;
    else
        mid_h2              <=          mat_row2_2;
end
          
always @(posedge sclk or negedge rst_n)
begin
    if(rst_n == 1'b0)
        min_h2              <=          18'd0;        
    else if(mat_row2[15:0] <= mat_row2_1[15:0] && mat_row2[15:0] <= mat_row2_2[15:0])
        min_h2              <=          mat_row2;
    else if(mat_row2_1[15:0] <= mat_row2[15:0] && mat_row2_1[15:0] <= mat_row2_2[15:0]) 
        min_h2              <=          mat_row2_1;
    else
        min_h2              <=          mat_row2_2;  
end
 
always @(posedge sclk or negedge rst_n)
begin
    if(rst_n == 1'b0)
        max_h3              <=          18'd0;        
    else if(mat_row3[15:0] >= mat_row3_1[15:0] && mat_row3[15:0] >= mat_row3_2[15:0])
        max_h3              <=          mat_row3;
    else if(mat_row3_1[15:0] >= mat_row3[15:0] && mat_row3_1[15:0] >= mat_row3_2[15:0]) 
        max_h3              <=          mat_row3_1;
    else
        max_h3              <=          mat_row3_2;
end
 
always @(posedge sclk or negedge rst_n)
begin
    if(rst_n == 1'b0)
        mid_h3              <=          18'd0;        
    else if((mat_row3[15:0] >= mat_row3_1[15:0] && mat_row3_1[15:0] >= mat_row3_2[15:0]) || (mat_row3_2[15:0] >= mat_row3_1[15:0] && mat_row3_1[15:0] >= mat_row3[15:0]))
        mid_h3              <=          mat_row3_1;
    else if((mat_row3_1[15:0] >= mat_row3[15:0] && mat_row3[15:0] >= mat_row3_2[15:0]) || (mat_row3_2[15:0] >= mat_row3[15:0] && mat_row3[15:0] >= mat_row3_1[15:0])) 
        mid_h3              <=          mat_row3;
    else
        mid_h3              <=          mat_row3_2;
end
 
          
always @(posedge sclk or negedge rst_n)
begin
    if(rst_n == 1'b0)
        min_h3              <=          18'd0;        
    else if(mat_row3[15:0] <= mat_row3_1[15:0] && mat_row3[15:0] <= mat_row3_2[15:0])
        min_h3              <=          mat_row3;
    else if(mat_row3_1[15:0] <= mat_row3[15:0] && mat_row3_1[15:0] <= mat_row3_2[15:0]) 
        min_h3              <=          mat_row3_1;
    else
        min_h3              <=          mat_row3_2;
end
 
 
always @(posedge sclk or negedge rst_n)
begin
    if(rst_n == 1'b0)
        min_max             <=          18'd0;
    else if(max_h1[15:0] <= max_h2[15:0] && max_h1[15:0] <= max_h3[15:0])
        min_max             <=          max_h1;
    else if(max_h2[15:0] <= max_h1[15:0] && max_h2[15:0] <= max_h3[15:0])
        min_max             <=          max_h2;
    else
        min_max             <=          max_h3;
end
          
   
always @(posedge sclk or negedge rst_n)
begin
    if(rst_n == 1'b0)
        mid_mid             <=          18'd0;
    else if((mid_h1[15:0] >= mid_h2[15:0] && mid_h2[15:0] >= mid_h3[15:0]) || (mid_h3[15:0] >= mid_h2[15:0] && mid_h2[15:0] >= mid_h1[15:0]))
        mid_mid             <=          mid_h2;
    else if((mid_h2[15:0] >= mid_h1[15:0] && mid_h1[15:0] >= mid_h3[15:0]) || (mid_h3[15:0] >= mid_h1[15:0] && mid_h1[15:0] >= mid_h2[15:0]))
        mid_mid             <=          mid_h1;
    else
        mid_mid             <=          mid_h3;
end
 
always @(posedge sclk or negedge rst_n)
begin
    if(rst_n == 1'b0)
        max_min             <=          18'd0;
    else if(min_h1[15:0] <= min_h2[15:0] && min_h1[15:0] <= min_h3[15:0]) 
        max_min             <=          min_h1;
    else if(min_h2[15:0] <= min_h1[15:0] && min_h2[15:0] <= min_h3[15:0])
        max_min             <=          min_h2;
    else
        max_min             <=          min_h3;
end
           
always @(posedge sclk or negedge rst_n)
begin
    if(rst_n == 1'b0)
      begin
        O_frame             <=          1'd0;
        O_line                <=            1'd0;
        O_data              <=          16'd0; 
      end
    else if((mid_mid[15:0] >= min_max[15:0] && min_max[15:0] >= max_min[15:0]) || (max_min[15:0] >= min_max[15:0] && min_max[15:0] >= mid_mid[15:0]))
      begin
        O_frame             <=          mat_row2_1[18];
        O_line                <=            mat_row2_1[17];
        if(mat_row2_1[16] == 1)
            O_data              <=          min_max[15:0];
        else
            O_data              <=          mat_row2_1[15:0];
      end
    else if((min_max[15:0] >= mid_mid[15:0] && mid_mid[15:0] >= max_min[15:0]) || (max_min[15:0] >= mid_mid[15:0] && mid_mid[15:0] >= min_max[15:0]))
      begin
        O_frame             <=          mat_row2_1[18];
        O_line                <=            mat_row2_1[17];
        if(mat_row2_1[16] == 1)
            O_data              <=          mid_mid[15:0];
        else
            O_data              <=          mat_row2_1[15:0];
      end
    else
      begin
        O_frame             <=          mat_row2_1[18];
        O_line              <=          mat_row2_1[17];
        if(mat_row2_1[16] == 1)
            O_data              <=          max_min[15:0];
        else
            O_data              <=          mat_row2_1[15:0];
      end
end
          
        
endmodule

以上两个模块构成了完整的坏点替换算法，可以用带有坏点的模拟图像输入至模块顶层即可输出替换后的结果，I_frame_valid为输入帧同步高有效，I_line_valid为输入行同步高有效，I_video_in为像素数据，I_clk为像素同步时钟输入这几个信号就可以得到输出结果。可以防止试试。

2.基于均值滤波的坏点替换模块

`timescale 1ns / 1ps
//
// Company: 
// Engineer: 
// 
// Create Date: 2022/01/06 10:48:23
// Design Name: 
// Module Name: blind_point
// Project Name: 
// Target Devices: 
// Tool Versions: 
// Description: 
//参考均值滤波法，使用两个深度为1280的19位fifo级联，延时前两行，每行输出端使用两个寄存器暂存前两个图像数据，形成3*3寄存器矩阵
//对3*3寄存器取均值后直接输出，可将行有效帧有效和盲元标志并入像素数据高三位，直接输入，计算结果直接输出不必做状态机，不必管输入的帧有效和行有效
// Dependencies: 
// 
// Revision:
// Revision 0.01 - File Created
// Additional Comments:
// 
//
 
 
module blind_point(
   input I_clk,
   input I_reset,
   input I_clk_test,
   input I_frame_valid,
   input I_line_valid,
   input[15:0] I_video_in,
   input[31:0] I_par_Q,
   output wire O_frame_valid,
   output wire O_line_valid,
   output wire[15:0] O_video_out,
   output O_hblank,
   output O_vblank
    );
    
reg [18:0] S_temp0_r0;
reg [18:0] S_temp0_r1;
reg [18:0] S_temp0_r2;
reg [18:0] S_temp1_r0;
reg [18:0] S_temp1_r1;
reg [18:0] S_temp1_r2;
reg [18:0] S_temp2_r0;
reg [18:0] S_temp2_r1;
reg [18:0] S_temp2_r2;
wire S_blind_sig;
wire[18:0] S_fifo0;
wire[18:0] S_fifo1;
wire [18:0] S_fifo0_1;
wire [18:0] S_fifo1_1;
wire [18:0] S_add;
wire [18:0] S_video_in;
wire [15:0] S_result;
wire S_frame_valid, S_line_valid;
wire [15:0] S_video_out;
assign S_blind_sig = I_par_Q[28];
assign S_video_in = {I_frame_valid,I_line_valid,S_blind_sig,I_video_in};
assign O_hblank = (~O_line_valid) & O_frame_valid;
assign O_vblank = ~O_frame_valid;
 
 
c_shift_ram_0 c_shift_ram_0_0(
    .D(S_video_in),
    .CLK(I_clk),
    .Q(S_fifo0_1)
  );
  
c_shift_ram_0 c_shift_ram_0_1(
    .D(S_fifo0_1),
    .CLK(I_clk),
    .Q(S_fifo0)
  );
  
c_shift_ram_0 c_shift_ram_0_2(
    .D(S_fifo0),
    .CLK(I_clk),
    .Q(S_fifo1_1)
  );
  
c_shift_ram_0 c_shift_ram_0_3(
    .D(S_fifo1_1),
    .CLK(I_clk),
    .Q(S_fifo1)
  );
 
 
///
always @(posedge I_clk or negedge I_reset)
begin
 if(!I_reset)
  begin
   S_temp0_r2 <= 0;
   S_temp0_r1 <= 0;
   S_temp0_r0 <= 0;
  end
 else
  begin
   S_temp0_r2 <= S_video_in;
   S_temp0_r1 <= S_temp0_r2;
   S_temp0_r0 <= S_temp0_r1;
  end
end
 
always @(posedge I_clk or negedge I_reset)
begin
 if(!I_reset)
  begin
   S_temp1_r2 <= 0;
   S_temp1_r1 <= 0;
   S_temp1_r0 <= 0;
  end
 else
  begin
   S_temp1_r2 <= S_fifo0;
   S_temp1_r1 <= S_temp1_r2;
   S_temp1_r0 <= S_temp1_r1;
  end
end
 
always @(posedge I_clk or negedge I_reset)
begin
 if(!I_reset)
  begin
   S_temp2_r2 <= 0;
   S_temp2_r1 <= 0;
   S_temp2_r0 <= 0;
  end
 else
  begin
   S_temp2_r2 <= S_fifo1;
   S_temp2_r1 <= S_temp2_r2;
   S_temp2_r0 <= S_temp2_r1;
  end
end
 
 
assign S_add = (S_temp0_r2[15:0] + S_temp0_r1[15:0] + S_temp0_r0[15:0])
                + (S_temp1_r2[15:0] + S_temp1_r0[15:0])
                + (S_temp2_r2[15:0] + S_temp2_r1[15:0] + S_temp2_r0[15:0]);
assign S_result =  S_add/8 ;              
assign O_video_out = (S_temp1_r1[16]) ? S_result : S_temp1_r1[15:0];
assign O_frame_valid = S_temp1_r1[18];
assign O_line_valid = S_temp1_r1[17];
 
 
endmodule

这个是用均值进行替换的3*3开窗坏点替换，可以直接输入图像试试。