The easiest way is to use HTML in your text, check How do I create a JLabel with the text underlined?

But here’s a code sample that does not make use of the HTML feature. It underlines all Integers in the JTable whose values are between 1970 and 1980.

Main.java:

import javax.swing.table.*;
import java.awt.event.*;
import javax.swing.*;
import java.awt.*;
 
public class Main extends JFrame 
{
   public Main() {
      super("Table example, Wines from Bordeaux");
 
      Object[][] tabledata = {
            { "Chateau Meyney, St. Estephe", 	   new Integer(1994), "$18.75"},
            { "Chateau Montrose, St. Estephe", 	   new Integer(1975), "$54.25" },
            { "Chateau Gloria, St. Julien", 	   new Integer(1993), "$22.99" },
            { "Chateau Beychevelle, St. Julien",   new Integer(1970), "$61.63" },
            { "Chateau La Tour de Mons, Margeaux", new Integer(1975), "$57.03" },
            { "Chateau Brane-Cantenac, Margeaux",  new Integer(1978), "$49.92" },
      };
 
      String columnheaders[] = { "Wine", "Vintage", "Price" };
 
      JTable table = new JTable(tabledata, columnheaders);
      table.setPreferredScrollableViewportSize(new Dimension(500, 70));
 
      UnderlineTableCellRenderer renderer = new UnderlineTableCellRenderer();
      table.setDefaultRenderer(Object.class, renderer);
      table.setRowHeight(30);
 
      JScrollPane scrollPane = new JScrollPane(table);
      getContentPane().add(scrollPane);
 
      addWindowListener(new WindowAdapter() {
         public void windowClosing(WindowEvent we) {
            System.exit(0);
         }
      });
 
      pack();
   }
 
   public static void main(String []args) {
      Main main = new Main();
      main.show();
   }
}
 
class UnderlineTableCellRenderer extends DefaultTableCellRenderer 
{
   public Component getTableCellRendererComponent(JTable table, Object value, boolean isSelected,
                                                  boolean hasFocus, int row, int column) {
      if (value instanceof Integer) {
         Integer amount = (Integer) value;
         if (amount.intValue() > 1970 && amount.intValue() < 1980) {
            return new UnderlinedJLabel("" + amount.intValue());
         }
      }
 
      return super.getTableCellRendererComponent
                      (table, value, isSelected, hasFocus, row, column);
   }
}
 
class UnderlinedJLabel extends JLabel
{
   public UnderlinedJLabel() {
   }
 
   public UnderlinedJLabel(Icon image) {
      super(image);
   }
 
   public UnderlinedJLabel(Icon image, int horizontalAlignment) {
      super(image, horizontalAlignment);
   }
 
   public UnderlinedJLabel(String text) {
      super(text);
   }
 
   public UnderlinedJLabel(String text, Icon icon, int horizontalAlignment) {
      super(text, icon, horizontalAlignment);
   }
 
   public UnderlinedJLabel(String text, int horizontalAlignment) {
      super(text, horizontalAlignment);
   }
 
   public void paint(Graphics g) {
      super.paint(g);
      underline(g);
   }
 
   protected void underline(Graphics g) {
      Insets insets = getInsets();
      FontMetrics fm = g.getFontMetrics();
      Rectangle textR = new Rectangle();
      Rectangle viewR = new Rectangle(
                                  insets.left, 
                                  insets.top, 
                                  getWidth() - (insets.right + insets.left), 
                                  getHeight() - (insets.bottom + insets.top));
  
      // compute and return location of the icons origin,
      // the location of the text baseline, and a possibly clipped
      // version of the compound label string.  Locations are computed 
      // relative to the viewR rectangle.
      String text = SwingUtilities.layoutCompoundLabel(
                         this,                        // this JLabel
                         fm,                          // current FontMetrics
                         getText(),                   // text
                         getIcon(),                   // icon
                         getVerticalAlignment(),      
                         getHorizontalAlignment(),
                         getVerticalTextPosition(),
                         getHorizontalTextPosition(), 
                         viewR,                       
                         new Rectangle(),             // don't care about icon rectangle
                         textR,                       // resulting text locations
                         getText() == null ? 0 : ((Integer)UIManager.get("Button.textIconGap")).intValue());
 
      // draw line
      int textShiftOffset = ((Integer) UIManager.get("Button.textShiftOffset")).intValue();
      g.fillRect(textR.x +
                 textShiftOffset - 4,
                 textR.y + fm.getAscent() + textShiftOffset + 2,
                 textR.width, 
                 1);
   }
}